sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ada.h"
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bio.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/taskqueue.h>
  42 #include <sys/lock.h>
  43 #include <sys/mutex.h>
  44 #include <sys/conf.h>
  45 #include <sys/devicestat.h>
  46 #include <sys/eventhandler.h>
  47 #include <sys/malloc.h>
  48 #include <sys/endian.h>
  49 #include <sys/cons.h>
  50 #include <sys/proc.h>
  51 #include <sys/reboot.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom.h>
  54 #include <geom/geom_disk.h>
  55 #endif /* _KERNEL */
  56
  57 #ifndef _KERNEL
  58 #include <stdio.h>
  59 #include <string.h>
  60 #endif /* _KERNEL */
  61
  62 #include <cam/cam.h>
  63 #include <cam/cam_ccb.h>
  64 #include <cam/cam_periph.h>
  65 #include <cam/cam_xpt_periph.h>
  66 #include <cam/scsi/scsi_all.h>
  67 #include <cam/scsi/scsi_da.h>
  68 #include <cam/cam_sim.h>
  69 #include <cam/cam_iosched.h>
  70
  71 #include <cam/ata/ata_all.h>
  72
  73 #include <machine/md_var.h>     /* geometry translation */
  74
  75 #ifdef _KERNEL
  76
  77 #define ATA_MAX_28BIT_LBA               268435455UL
  78
  79 extern int iosched_debug;
  80
  81 typedef enum {
  82         ADA_STATE_RAHEAD,
  83         ADA_STATE_WCACHE,
  84         ADA_STATE_LOGDIR,
  85         ADA_STATE_IDDIR,
  86         ADA_STATE_SUP_CAP,
  87         ADA_STATE_ZONE,
  88         ADA_STATE_NORMAL
  89 } ada_state;
  90
  91 typedef enum {
  92         ADA_FLAG_CAN_48BIT      = 0x00000002,
  93         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  94         ADA_FLAG_CAN_NCQ        = 0x00000008,
  95         ADA_FLAG_CAN_DMA        = 0x00000010,
  96         ADA_FLAG_NEED_OTAG      = 0x00000020,
  97         ADA_FLAG_WAS_OTAG       = 0x00000040,
  98         ADA_FLAG_CAN_TRIM       = 0x00000080,
  99         ADA_FLAG_OPEN           = 0x00000100,
 100         ADA_FLAG_SCTX_INIT      = 0x00000200,
 101         ADA_FLAG_CAN_CFA        = 0x00000400,
 102         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 103         ADA_FLAG_CAN_DMA48      = 0x00001000,
 104         ADA_FLAG_CAN_LOG        = 0x00002000,
 105         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 106         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 107         ADA_FLAG_CAN_ZONE       = 0x00010000,
 108         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 109         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 110         ADA_FLAG_PROBED         = 0x00080000,
 111         ADA_FLAG_ANNOUNCED      = 0x00100000,
 112         ADA_FLAG_DIRTY          = 0x00200000,
 113         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 114         ADA_FLAG_PIM_ATA_EXT    = 0x00800000
 115 } ada_flags;
 116
 117 typedef enum {
 118         ADA_Q_NONE              = 0x00,
 119         ADA_Q_4K                = 0x01,
 120         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 121         ADA_Q_LOG_BROKEN        = 0x04,
 122         ADA_Q_SMR_DM            = 0x08,
 123         ADA_Q_NO_TRIM           = 0x10,
 124         ADA_Q_128KB             = 0x20
 125 } ada_quirks;
 126
 127 #define ADA_Q_BIT_STRING        \
 128         "\020"                  \
 129         "\0014K"                \
 130         "\002NCQ_TRIM_BROKEN"   \
 131         "\003LOG_BROKEN"        \
 132         "\004SMR_DM"            \
 133         "\005NO_TRIM"           \
 134         "\006128KB"
 135
 136 typedef enum {
 137         ADA_CCB_RAHEAD          = 0x01,
 138         ADA_CCB_WCACHE          = 0x02,
 139         ADA_CCB_BUFFER_IO       = 0x03,
 140         ADA_CCB_DUMP            = 0x05,
 141         ADA_CCB_TRIM            = 0x06,
 142         ADA_CCB_LOGDIR          = 0x07,
 143         ADA_CCB_IDDIR           = 0x08,
 144         ADA_CCB_SUP_CAP         = 0x09,
 145         ADA_CCB_ZONE            = 0x0a,
 146         ADA_CCB_TYPE_MASK       = 0x0F,
 147 } ada_ccb_state;
 148
 149 typedef enum {
 150         ADA_ZONE_NONE           = 0x00,
 151         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 152         ADA_ZONE_HOST_AWARE     = 0x02,
 153         ADA_ZONE_HOST_MANAGED   = 0x03
 154 } ada_zone_mode;
 155
 156 typedef enum {
 157         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 158         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 159         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 160         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 161         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 162         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 163                                            ADA_ZONE_FLAG_OPEN_SUP |
 164                                            ADA_ZONE_FLAG_CLOSE_SUP |
 165                                            ADA_ZONE_FLAG_FINISH_SUP |
 166                                            ADA_ZONE_FLAG_RWP_SUP),
 167         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 168         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 169         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 170         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 171         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 172                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 173                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 174 } ada_zone_flags;
 175
 176 static struct ada_zone_desc {
 177         ada_zone_flags value;
 178         const char *desc;
 179 } ada_zone_desc_table[] = {
 180         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 181         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 182         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 183         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 184         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 185 };
 186
 187
 188 /* Offsets into our private area for storing information */
 189 #define ccb_state       ppriv_field0
 190 #define ccb_bp          ppriv_ptr1
 191
 192 typedef enum {
 193         ADA_DELETE_NONE,
 194         ADA_DELETE_DISABLE,
 195         ADA_DELETE_CFA_ERASE,
 196         ADA_DELETE_DSM_TRIM,
 197         ADA_DELETE_NCQ_DSM_TRIM,
 198         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 199         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 200 } ada_delete_methods;
 201
 202 static const char *ada_delete_method_names[] =
 203     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 204 #if 0
 205 static const char *ada_delete_method_desc[] =
 206     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 207 #endif
 208
 209 struct disk_params {
 210         u_int8_t  heads;
 211         u_int8_t  secs_per_track;
 212         u_int32_t cylinders;
 213         u_int32_t secsize;      /* Number of bytes/logical sector */
 214         u_int64_t sectors;      /* Total number sectors */
 215 };
 216
 217 #define TRIM_MAX_BLOCKS 8
 218 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 219 struct trim_request {
 220         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 221         TAILQ_HEAD(, bio) bps;
 222 };
 223
 224 struct ada_softc {
 225         struct   cam_iosched_softc *cam_iosched;
 226         int      outstanding_cmds;      /* Number of active commands */
 227         int      refcount;              /* Active xpt_action() calls */
 228         ada_state state;
 229         ada_flags flags;
 230         ada_zone_mode zone_mode;
 231         ada_zone_flags zone_flags;
 232         struct ata_gp_log_dir ata_logdir;
 233         int valid_logdir_len;
 234         struct ata_identify_log_pages ata_iddir;
 235         int valid_iddir_len;
 236         uint64_t optimal_seq_zones;
 237         uint64_t optimal_nonseq_zones;
 238         uint64_t max_seq_zones;
 239         ada_quirks quirks;
 240         ada_delete_methods delete_method;
 241         int      trim_max_ranges;
 242         int      read_ahead;
 243         int      write_cache;
 244         int      unmappedio;
 245         int      rotating;
 246 #ifdef CAM_TEST_FAILURE
 247         int      force_read_error;
 248         int      force_write_error;
 249         int      periodic_read_error;
 250         int      periodic_read_count;
 251 #endif
 252         struct ccb_pathinq      cpi;
 253         struct disk_params      params;
 254         struct disk             *disk;
 255         struct task             sysctl_task;
 256         struct sysctl_ctx_list  sysctl_ctx;
 257         struct sysctl_oid       *sysctl_tree;
 258         struct callout          sendordered_c;
 259         struct trim_request     trim_req;
 260         uint64_t                trim_count;
 261         uint64_t                trim_ranges;
 262         uint64_t                trim_lbas;
 263 #ifdef CAM_IO_STATS
 264         struct sysctl_ctx_list  sysctl_stats_ctx;
 265         struct sysctl_oid       *sysctl_stats_tree;
 266         u_int   timeouts;
 267         u_int   errors;
 268         u_int   invalidations;
 269 #endif
 270 #define ADA_ANNOUNCETMP_SZ 80
 271         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 272 #define ADA_ANNOUNCE_SZ 400
 273         char    announce_buffer[ADA_ANNOUNCE_SZ];
 274 };
 275
 276 struct ada_quirk_entry {
 277         struct scsi_inquiry_pattern inq_pat;
 278         ada_quirks quirks;
 279 };
 280
 281 static struct ada_quirk_entry ada_quirk_table[] =
 282 {
 283         {
 284                 /* Sandisk X400 */
 285                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SanDisk?SD8SB8U1T00*", "X4162000*" },
 286                 /*quirks*/ADA_Q_128KB
 287         },
 288         {
 289                 /* Hitachi Advanced Format (4k) drives */
 290                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 291                 /*quirks*/ADA_Q_4K
 292         },
 293         {
 294                 /* Samsung Advanced Format (4k) drives */
 295                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 296                 /*quirks*/ADA_Q_4K
 297         },
 298         {
 299                 /* Samsung Advanced Format (4k) drives */
 300                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 301                 /*quirks*/ADA_Q_4K
 302         },
 303         {
 304                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 305                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 306                 /*quirks*/ADA_Q_4K
 307         },
 308         {
 309                 /* Seagate Barracuda Advanced Format (4k) drives */
 310                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 311                 /*quirks*/ADA_Q_4K
 312         },
 313         {
 314                 /* Seagate Barracuda Advanced Format (4k) drives */
 315                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 316                 /*quirks*/ADA_Q_4K
 317         },
 318         {
 319                 /* Seagate Momentus Advanced Format (4k) drives */
 320                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 321                 /*quirks*/ADA_Q_4K
 322         },
 323         {
 324                 /* Seagate Momentus Advanced Format (4k) drives */
 325                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 326                 /*quirks*/ADA_Q_4K
 327         },
 328         {
 329                 /* Seagate Momentus Advanced Format (4k) drives */
 330                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 331                 /*quirks*/ADA_Q_4K
 332         },
 333         {
 334                 /* Seagate Momentus Advanced Format (4k) drives */
 335                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 336                 /*quirks*/ADA_Q_4K
 337         },
 338         {
 339                 /* Seagate Momentus Advanced Format (4k) drives */
 340                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 341                 /*quirks*/ADA_Q_4K
 342         },
 343         {
 344                 /* Seagate Momentus Advanced Format (4k) drives */
 345                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 346                 /*quirks*/ADA_Q_4K
 347         },
 348         {
 349                 /* Seagate Momentus Advanced Format (4k) drives */
 350                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 351                 /*quirks*/ADA_Q_4K
 352         },
 353         {
 354                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 355                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 356                 /*quirks*/ADA_Q_4K
 357         },
 358         {
 359                 /* WDC Caviar Red Advanced Format (4k) drives */
 360                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 361                 /*quirks*/ADA_Q_4K
 362         },
 363         {
 364                 /* WDC Caviar Green Advanced Format (4k) drives */
 365                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 366                 /*quirks*/ADA_Q_4K
 367         },
 368         {
 369                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 370                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 371                 /*quirks*/ADA_Q_4K
 372         },
 373         {
 374                 /* WDC Caviar Red Advanced Format (4k) drives */
 375                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 376                 /*quirks*/ADA_Q_4K
 377         },
 378         {
 379                 /* WDC Caviar Black Advanced Format (4k) drives */
 380                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 381                 /*quirks*/ADA_Q_4K
 382         },
 383         {
 384                 /* WDC Caviar Black Advanced Format (4k) drives */
 385                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 386                 /*quirks*/ADA_Q_4K
 387         },
 388         {
 389                 /* WDC Caviar Green Advanced Format (4k) drives */
 390                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 391                 /*quirks*/ADA_Q_4K
 392         },
 393         {
 394                 /* WDC Caviar Green Advanced Format (4k) drives */
 395                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 396                 /*quirks*/ADA_Q_4K
 397         },
 398         {
 399                 /* WDC Scorpio Black Advanced Format (4k) drives */
 400                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 401                 /*quirks*/ADA_Q_4K
 402         },
 403         {
 404                 /* WDC Scorpio Black Advanced Format (4k) drives */
 405                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 406                 /*quirks*/ADA_Q_4K
 407         },
 408         {
 409                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 410                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 411                 /*quirks*/ADA_Q_4K
 412         },
 413         {
 414                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 415                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 416                 /*quirks*/ADA_Q_4K
 417         },
 418         /* SSDs */
 419         {
 420                 /*
 421                  * Corsair Force 2 SSDs
 422                  * 4k optimised & trim only works in 4k requests + 4k aligned
 423                  */
 424                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 425                 /*quirks*/ADA_Q_4K
 426         },
 427         {
 428                 /*
 429                  * Corsair Force 3 SSDs
 430                  * 4k optimised & trim only works in 4k requests + 4k aligned
 431                  */
 432                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 433                 /*quirks*/ADA_Q_4K
 434         },
 435         {
 436                 /*
 437                  * Corsair Neutron GTX SSDs
 438                  * 4k optimised & trim only works in 4k requests + 4k aligned
 439                  */
 440                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 441                 /*quirks*/ADA_Q_4K
 442         },
 443         {
 444                 /*
 445                  * Corsair Force GT & GS SSDs
 446                  * 4k optimised & trim only works in 4k requests + 4k aligned
 447                  */
 448                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 449                 /*quirks*/ADA_Q_4K
 450         },
 451         {
 452                 /*
 453                  * Crucial M4 SSDs
 454                  * 4k optimised & trim only works in 4k requests + 4k aligned
 455                  */
 456                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 457                 /*quirks*/ADA_Q_4K
 458         },
 459         {
 460                 /*
 461                  * Crucial M500 SSDs MU07 firmware
 462                  * NCQ Trim works
 463                  */
 464                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 465                 /*quirks*/0
 466         },
 467         {
 468                 /*
 469                  * Crucial M500 SSDs all other firmware
 470                  * NCQ Trim doesn't work
 471                  */
 472                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 473                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 474         },
 475         {
 476                 /*
 477                  * Crucial M550 SSDs
 478                  * NCQ Trim doesn't work, but only on MU01 firmware
 479                  */
 480                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 481                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 482         },
 483         {
 484                 /*
 485                  * Crucial MX100 SSDs
 486                  * NCQ Trim doesn't work, but only on MU01 firmware
 487                  */
 488                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 489                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 490         },
 491         {
 492                 /*
 493                  * Crucial RealSSD C300 SSDs
 494                  * 4k optimised
 495                  */
 496                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 497                 "*" }, /*quirks*/ADA_Q_4K
 498         },
 499         {
 500                 /*
 501                  * FCCT M500 SSDs
 502                  * NCQ Trim doesn't work
 503                  */
 504                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 505                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 506         },
 507         {
 508                 /*
 509                  * Intel 320 Series SSDs
 510                  * 4k optimised & trim only works in 4k requests + 4k aligned
 511                  */
 512                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 513                 /*quirks*/ADA_Q_4K
 514         },
 515         {
 516                 /*
 517                  * Intel 330 Series SSDs
 518                  * 4k optimised & trim only works in 4k requests + 4k aligned
 519                  */
 520                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 521                 /*quirks*/ADA_Q_4K
 522         },
 523         {
 524                 /*
 525                  * Intel 510 Series SSDs
 526                  * 4k optimised & trim only works in 4k requests + 4k aligned
 527                  */
 528                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 529                 /*quirks*/ADA_Q_4K
 530         },
 531         {
 532                 /*
 533                  * Intel 520 Series SSDs
 534                  * 4k optimised & trim only works in 4k requests + 4k aligned
 535                  */
 536                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 537                 /*quirks*/ADA_Q_4K
 538         },
 539         {
 540                 /*
 541                  * Intel S3610 Series SSDs
 542                  * 4k optimised & trim only works in 4k requests + 4k aligned
 543                  */
 544                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 545                 /*quirks*/ADA_Q_4K
 546         },
 547         {
 548                 /*
 549                  * Intel X25-M Series SSDs
 550                  * 4k optimised & trim only works in 4k requests + 4k aligned
 551                  */
 552                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 553                 /*quirks*/ADA_Q_4K
 554         },
 555         {
 556                 /*
 557                  * KingDian S200 60GB P0921B
 558                  * Trimming crash the SSD
 559                  */
 560                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KingDian S200 *", "*" },
 561                 /*quirks*/ADA_Q_NO_TRIM
 562         },
 563         {
 564                 /*
 565                  * Kingston E100 Series SSDs
 566                  * 4k optimised & trim only works in 4k requests + 4k aligned
 567                  */
 568                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 569                 /*quirks*/ADA_Q_4K
 570         },
 571         {
 572                 /*
 573                  * Kingston HyperX 3k SSDs
 574                  * 4k optimised & trim only works in 4k requests + 4k aligned
 575                  */
 576                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 577                 /*quirks*/ADA_Q_4K
 578         },
 579         {
 580                 /*
 581                  * Marvell SSDs (entry taken from OpenSolaris)
 582                  * 4k optimised & trim only works in 4k requests + 4k aligned
 583                  */
 584                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 585                 /*quirks*/ADA_Q_4K
 586         },
 587         {
 588                 /*
 589                  * Micron M500 SSDs firmware MU07
 590                  * NCQ Trim works?
 591                  */
 592                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 593                 /*quirks*/0
 594         },
 595         {
 596                 /*
 597                  * Micron M500 SSDs all other firmware
 598                  * NCQ Trim doesn't work
 599                  */
 600                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 601                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 602         },
 603         {
 604                 /*
 605                  * Micron M5[15]0 SSDs
 606                  * NCQ Trim doesn't work, but only MU01 firmware
 607                  */
 608                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 609                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 610         },
 611         {
 612                 /*
 613                  * Micron 5100 SSDs
 614                  * 4k optimised & trim only works in 4k requests + 4k aligned
 615                  */
 616                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 617                 /*quirks*/ADA_Q_4K
 618         },
 619         {
 620                 /*
 621                  * OCZ Agility 2 SSDs
 622                  * 4k optimised & trim only works in 4k requests + 4k aligned
 623                  */
 624                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 625                 /*quirks*/ADA_Q_4K
 626         },
 627         {
 628                 /*
 629                  * OCZ Agility 3 SSDs
 630                  * 4k optimised & trim only works in 4k requests + 4k aligned
 631                  */
 632                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 633                 /*quirks*/ADA_Q_4K
 634         },
 635         {
 636                 /*
 637                  * OCZ Deneva R Series SSDs
 638                  * 4k optimised & trim only works in 4k requests + 4k aligned
 639                  */
 640                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 641                 /*quirks*/ADA_Q_4K
 642         },
 643         {
 644                 /*
 645                  * OCZ Vertex 2 SSDs (inc pro series)
 646                  * 4k optimised & trim only works in 4k requests + 4k aligned
 647                  */
 648                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 649                 /*quirks*/ADA_Q_4K
 650         },
 651         {
 652                 /*
 653                  * OCZ Vertex 3 SSDs
 654                  * 4k optimised & trim only works in 4k requests + 4k aligned
 655                  */
 656                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 657                 /*quirks*/ADA_Q_4K
 658         },
 659         {
 660                 /*
 661                  * OCZ Vertex 4 SSDs
 662                  * 4k optimised & trim only works in 4k requests + 4k aligned
 663                  */
 664                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 665                 /*quirks*/ADA_Q_4K
 666         },
 667         {
 668                 /*
 669                  * Samsung 750 SSDs
 670                  * 4k optimised, NCQ TRIM seems to work
 671                  */
 672                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 673                 /*quirks*/ADA_Q_4K
 674         },
 675         {
 676                 /*
 677                  * Samsung 830 Series SSDs
 678                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 679                  */
 680                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 681                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 682         },
 683         {
 684                 /*
 685                  * Samsung 840 SSDs
 686                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 687                  */
 688                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 689                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 690         },
 691         {
 692                 /*
 693                  * Samsung 845 SSDs
 694                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 695                  */
 696                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 697                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 698         },
 699         {
 700                 /*
 701                  * Samsung 850 SSDs
 702                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 703                  */
 704                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 705                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 706         },
 707         {
 708                 /*
 709                  * Samsung SM863 Series SSDs (MZ7KM*)
 710                  * 4k optimised, NCQ believed to be working
 711                  */
 712                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 713                 /*quirks*/ADA_Q_4K
 714         },
 715         {
 716                 /*
 717                  * Samsung 843T Series SSDs (MZ7WD*)
 718                  * Samsung PM851 Series SSDs (MZ7TE*)
 719                  * Samsung PM853T Series SSDs (MZ7GE*)
 720                  * 4k optimised, NCQ believed to be broken since these are
 721                  * appear to be built with the same controllers as the 840/850.
 722                  */
 723                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 724                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 725         },
 726         {
 727                 /*
 728                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 729                  * starting with MZ7* too
 730                  */
 731                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 732                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 733         },
 734         {
 735                 /*
 736                  * Samsung PM851 Series SSDs Dell OEM
 737                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 738                  * 4k optimised, NCQ broken
 739                  */
 740                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 741                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 742         },
 743         {
 744                 /*
 745                  * SuperTalent TeraDrive CT SSDs
 746                  * 4k optimised & trim only works in 4k requests + 4k aligned
 747                  */
 748                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 749                 /*quirks*/ADA_Q_4K
 750         },
 751         {
 752                 /*
 753                  * XceedIOPS SATA SSDs
 754                  * 4k optimised
 755                  */
 756                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 757                 /*quirks*/ADA_Q_4K
 758         },
 759         {
 760                 /*
 761                  * Samsung drive that doesn't support READ LOG EXT or
 762                  * READ LOG DMA EXT, despite reporting that it does in
 763                  * ATA identify data:
 764                  * SAMSUNG HD200HJ KF100-06
 765                  */
 766                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 767                 /*quirks*/ADA_Q_LOG_BROKEN
 768         },
 769         {
 770                 /*
 771                  * Samsung drive that doesn't support READ LOG EXT or
 772                  * READ LOG DMA EXT, despite reporting that it does in
 773                  * ATA identify data:
 774                  * SAMSUNG HD501LJ CR100-10
 775                  */
 776                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 777                 /*quirks*/ADA_Q_LOG_BROKEN
 778         },
 779         {
 780                 /*
 781                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 782                  * Drive Managed SATA hard drive.  This drive doesn't report
 783                  * in firmware that it is a drive managed SMR drive.
 784                  */
 785                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 786                 /*quirks*/ADA_Q_SMR_DM
 787         },
 788         {
 789                 /* Default */
 790                 {
 791                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 792                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 793                 },
 794                 /*quirks*/0
 795         },
 796 };
 797
 798 static  disk_strategy_t adastrategy;
 799 static  dumper_t        adadump;
 800 static  periph_init_t   adainit;
 801 static  void            adadiskgonecb(struct disk *dp);
 802 static  periph_oninv_t  adaoninvalidate;
 803 static  periph_dtor_t   adacleanup;
 804 static  void            adaasync(void *callback_arg, u_int32_t code,
 805                                 struct cam_path *path, void *arg);
 806 static  int             adazonemodesysctl(SYSCTL_HANDLER_ARGS);
 807 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 808 static  void            adasysctlinit(void *context, int pending);
 809 static  int             adagetattr(struct bio *bp);
 810 static  void            adasetflags(struct ada_softc *softc,
 811                                     struct ccb_getdev *cgd);
 812 static void             adasetgeom(struct ada_softc *softc,
 813                                    struct ccb_getdev *cgd);
 814 static  periph_ctor_t   adaregister;
 815 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 816                                     struct ccb_ataio *ataio);
 817 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 818                                      struct ccb_ataio *ataio);
 819 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 820 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 821                                      struct bio *bp, int *queue_ccb);
 822 static  periph_start_t  adastart;
 823 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 824 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 825 static  void            adadone(struct cam_periph *periph,
 826                                union ccb *done_ccb);
 827 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 828                                 u_int32_t sense_flags);
 829 static callout_func_t   adasendorderedtag;
 830 static void             adashutdown(void *arg, int howto);
 831 static void             adasuspend(void *arg);
 832 static void             adaresume(void *arg);
 833
 834 #ifndef ADA_DEFAULT_TIMEOUT
 835 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 836 #endif
 837
 838 #ifndef ADA_DEFAULT_RETRY
 839 #define ADA_DEFAULT_RETRY       4
 840 #endif
 841
 842 #ifndef ADA_DEFAULT_SEND_ORDERED
 843 #define ADA_DEFAULT_SEND_ORDERED        1
 844 #endif
 845
 846 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 847 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 848 #endif
 849
 850 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 851 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 852 #endif
 853
 854 #ifndef ADA_DEFAULT_READ_AHEAD
 855 #define ADA_DEFAULT_READ_AHEAD  1
 856 #endif
 857
 858 #ifndef ADA_DEFAULT_WRITE_CACHE
 859 #define ADA_DEFAULT_WRITE_CACHE 1
 860 #endif
 861
 862 #define ADA_RA  (softc->read_ahead >= 0 ? \
 863                  softc->read_ahead : ada_read_ahead)
 864 #define ADA_WC  (softc->write_cache >= 0 ? \
 865                  softc->write_cache : ada_write_cache)
 866
 867 /*
 868  * Most platforms map firmware geometry to actual, but some don't.  If
 869  * not overridden, default to nothing.
 870  */
 871 #ifndef ata_disk_firmware_geom_adjust
 872 #define ata_disk_firmware_geom_adjust(disk)
 873 #endif
 874
 875 static int ada_retry_count = ADA_DEFAULT_RETRY;
 876 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 877 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 878 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 879 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 880 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 881 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 882 static int ada_enable_biospeedup = 1;
 883
 884 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 885     "CAM Direct Access Disk driver");
 886 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 887            &ada_retry_count, 0, "Normal I/O retry count");
 888 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 889            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 890 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 891            &ada_send_ordered, 0, "Send Ordered Tags");
 892 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 893            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 894 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 895            &ada_spindown_suspend, 0, "Spin down upon suspend");
 896 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 897            &ada_read_ahead, 0, "Enable disk read-ahead");
 898 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 899            &ada_write_cache, 0, "Enable disk write cache");
 900 SYSCTL_INT(_kern_cam_ada, OID_AUTO, enable_biospeedup, CTLFLAG_RDTUN,
 901            &ada_enable_biospeedup, 0, "Enable BIO_SPEEDUP processing");
 902
 903 /*
 904  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 905  * to the default timeout, we check to see whether an ordered
 906  * tagged transaction is appropriate to prevent simple tag
 907  * starvation.  Since we'd like to ensure that there is at least
 908  * 1/2 of the timeout length left for a starved transaction to
 909  * complete after we've sent an ordered tag, we must poll at least
 910  * four times in every timeout period.  This takes care of the worst
 911  * case where a starved transaction starts during an interval that
 912  * meets the requirement "don't send an ordered tag" test so it takes
 913  * us two intervals to determine that a tag must be sent.
 914  */
 915 #ifndef ADA_ORDEREDTAG_INTERVAL
 916 #define ADA_ORDEREDTAG_INTERVAL 4
 917 #endif
 918
 919 static struct periph_driver adadriver =
 920 {
 921         adainit, "ada",
 922         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 923 };
 924
 925 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 926
 927 PERIPHDRIVER_DECLARE(ada, adadriver);
 928
 929 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 930
 931 static int
 932 adaopen(struct disk *dp)
 933 {
 934         struct cam_periph *periph;
 935         struct ada_softc *softc;
 936         int error;
 937
 938         periph = (struct cam_periph *)dp->d_drv1;
 939         if (cam_periph_acquire(periph) != 0) {
 940                 return(ENXIO);
 941         }
 942
 943         cam_periph_lock(periph);
 944         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 945                 cam_periph_unlock(periph);
 946                 cam_periph_release(periph);
 947                 return (error);
 948         }
 949
 950         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 951             ("adaopen\n"));
 952
 953         softc = (struct ada_softc *)periph->softc;
 954         softc->flags |= ADA_FLAG_OPEN;
 955
 956         cam_periph_unhold(periph);
 957         cam_periph_unlock(periph);
 958         return (0);
 959 }
 960
 961 static int
 962 adaclose(struct disk *dp)
 963 {
 964         struct  cam_periph *periph;
 965         struct  ada_softc *softc;
 966         union ccb *ccb;
 967         int error;
 968
 969         periph = (struct cam_periph *)dp->d_drv1;
 970         softc = (struct ada_softc *)periph->softc;
 971         cam_periph_lock(periph);
 972
 973         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 974             ("adaclose\n"));
 975
 976         /* We only sync the cache if the drive is capable of it. */
 977         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 978             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 979             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 980             cam_periph_hold(periph, PRIBIO) == 0) {
 981
 982                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 983                 cam_fill_ataio(&ccb->ataio,
 984                                     1,
 985                                     NULL,
 986                                     CAM_DIR_NONE,
 987                                     0,
 988                                     NULL,
 989                                     0,
 990                                     ada_default_timeout*1000);
 991
 992                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 993                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 994                 else
 995                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 996                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 997                     /*sense_flags*/0, softc->disk->d_devstat);
 998
 999                 if (error != 0)
1000                         xpt_print(periph->path, "Synchronize cache failed\n");
1001                 softc->flags &= ~ADA_FLAG_DIRTY;
1002                 xpt_release_ccb(ccb);
1003                 cam_periph_unhold(periph);
1004         }
1005
1006         softc->flags &= ~ADA_FLAG_OPEN;
1007
1008         while (softc->refcount != 0)
1009                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
1010         cam_periph_unlock(periph);
1011         cam_periph_release(periph);
1012         return (0);
1013 }
1014
1015 static void
1016 adaschedule(struct cam_periph *periph)
1017 {
1018         struct ada_softc *softc = (struct ada_softc *)periph->softc;
1019
1020         if (softc->state != ADA_STATE_NORMAL)
1021                 return;
1022
1023         cam_iosched_schedule(softc->cam_iosched, periph);
1024 }
1025
1026 /*
1027  * Actually translate the requested transfer into one the physical driver
1028  * can understand.  The transfer is described by a buf and will include
1029  * only one physical transfer.
1030  */
1031 static void
1032 adastrategy(struct bio *bp)
1033 {
1034         struct cam_periph *periph;
1035         struct ada_softc *softc;
1036
1037         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1038         softc = (struct ada_softc *)periph->softc;
1039
1040         cam_periph_lock(periph);
1041
1042         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1043
1044         /*
1045          * If the device has been made invalid, error out
1046          */
1047         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1048                 cam_periph_unlock(periph);
1049                 biofinish(bp, NULL, ENXIO);
1050                 return;
1051         }
1052
1053         /*
1054          * Zone commands must be ordered, because they can depend on the
1055          * effects of previously issued commands, and they may affect
1056          * commands after them.
1057          */
1058         if (bp->bio_cmd == BIO_ZONE)
1059                 bp->bio_flags |= BIO_ORDERED;
1060
1061         /*
1062          * Place it in the queue of disk activities for this disk
1063          */
1064         cam_iosched_queue_work(softc->cam_iosched, bp);
1065
1066         /*
1067          * Schedule ourselves for performing the work.
1068          */
1069         adaschedule(periph);
1070         cam_periph_unlock(periph);
1071
1072         return;
1073 }
1074
1075 static int
1076 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1077 {
1078         struct      cam_periph *periph;
1079         struct      ada_softc *softc;
1080         u_int       secsize;
1081         struct      ccb_ataio ataio;
1082         struct      disk *dp;
1083         uint64_t    lba;
1084         uint16_t    count;
1085         int         error = 0;
1086
1087         dp = arg;
1088         periph = dp->d_drv1;
1089         softc = (struct ada_softc *)periph->softc;
1090         secsize = softc->params.secsize;
1091         lba = offset / secsize;
1092         count = length / secsize;
1093         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1094                 return (ENXIO);
1095
1096         memset(&ataio, 0, sizeof(ataio));
1097         if (length > 0) {
1098                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1099                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1100                 cam_fill_ataio(&ataio,
1101                     0,
1102                     NULL,
1103                     CAM_DIR_OUT,
1104                     0,
1105                     (u_int8_t *) virtual,
1106                     length,
1107                     ada_default_timeout*1000);
1108                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1109                     (lba + count >= ATA_MAX_28BIT_LBA ||
1110                     count >= 256)) {
1111                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1112                             0, lba, count);
1113                 } else {
1114                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1115                             0, lba, count);
1116                 }
1117                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1118                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1119                 if (error != 0)
1120                         printf("Aborting dump due to I/O error.\n");
1121
1122                 return (error);
1123         }
1124
1125         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1126                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1127
1128                 /*
1129                  * Tell the drive to flush its internal cache. if we
1130                  * can't flush in 5s we have big problems. No need to
1131                  * wait the default 60s to detect problems.
1132                  */
1133                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1134                 cam_fill_ataio(&ataio,
1135                                     0,
1136                                     NULL,
1137                                     CAM_DIR_NONE,
1138                                     0,
1139                                     NULL,
1140                                     0,
1141                                     5*1000);
1142
1143                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1144                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1145                 else
1146                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1147                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1148                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1149                 if (error != 0)
1150                         xpt_print(periph->path, "Synchronize cache failed\n");
1151         }
1152         return (error);
1153 }
1154
1155 static void
1156 adainit(void)
1157 {
1158         cam_status status;
1159
1160         /*
1161          * Install a global async callback.  This callback will
1162          * receive async callbacks like "new device found".
1163          */
1164         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1165
1166         if (status != CAM_REQ_CMP) {
1167                 printf("ada: Failed to attach master async callback "
1168                        "due to status 0x%x!\n", status);
1169         } else if (ada_send_ordered) {
1170
1171                 /* Register our event handlers */
1172                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1173                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1174                     printf("adainit: power event registration failed!\n");
1175                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1176                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1177                     printf("adainit: power event registration failed!\n");
1178                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1179                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1180                     printf("adainit: shutdown event registration failed!\n");
1181         }
1182 }
1183
1184 /*
1185  * Callback from GEOM, called when it has finished cleaning up its
1186  * resources.
1187  */
1188 static void
1189 adadiskgonecb(struct disk *dp)
1190 {
1191         struct cam_periph *periph;
1192
1193         periph = (struct cam_periph *)dp->d_drv1;
1194
1195         cam_periph_release(periph);
1196 }
1197
1198 static void
1199 adaoninvalidate(struct cam_periph *periph)
1200 {
1201         struct ada_softc *softc;
1202
1203         softc = (struct ada_softc *)periph->softc;
1204
1205         /*
1206          * De-register any async callbacks.
1207          */
1208         xpt_register_async(0, adaasync, periph, periph->path);
1209 #ifdef CAM_IO_STATS
1210         softc->invalidations++;
1211 #endif
1212
1213         /*
1214          * Return all queued I/O with ENXIO.
1215          * XXX Handle any transactions queued to the card
1216          *     with XPT_ABORT_CCB.
1217          */
1218         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1219
1220         disk_gone(softc->disk);
1221 }
1222
1223 static void
1224 adacleanup(struct cam_periph *periph)
1225 {
1226         struct ada_softc *softc;
1227
1228         softc = (struct ada_softc *)periph->softc;
1229
1230         cam_periph_unlock(periph);
1231
1232         cam_iosched_fini(softc->cam_iosched);
1233
1234         /*
1235          * If we can't free the sysctl tree, oh well...
1236          */
1237         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1238 #ifdef CAM_IO_STATS
1239                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1240                         xpt_print(periph->path,
1241                             "can't remove sysctl stats context\n");
1242 #endif
1243                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1244                         xpt_print(periph->path,
1245                             "can't remove sysctl context\n");
1246         }
1247
1248         disk_destroy(softc->disk);
1249         callout_drain(&softc->sendordered_c);
1250         free(softc, M_DEVBUF);
1251         cam_periph_lock(periph);
1252 }
1253
1254 static void
1255 adasetdeletemethod(struct ada_softc *softc)
1256 {
1257
1258         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1259                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1260         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1261                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1262         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1263                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1264         else
1265                 softc->delete_method = ADA_DELETE_NONE;
1266 }
1267
1268 static void
1269 adaasync(void *callback_arg, u_int32_t code,
1270         struct cam_path *path, void *arg)
1271 {
1272         struct ccb_getdev cgd;
1273         struct cam_periph *periph;
1274         struct ada_softc *softc;
1275
1276         periph = (struct cam_periph *)callback_arg;
1277         switch (code) {
1278         case AC_FOUND_DEVICE:
1279         {
1280                 struct ccb_getdev *cgd;
1281                 cam_status status;
1282
1283                 cgd = (struct ccb_getdev *)arg;
1284                 if (cgd == NULL)
1285                         break;
1286
1287                 if (cgd->protocol != PROTO_ATA)
1288                         break;
1289
1290                 /*
1291                  * Allocate a peripheral instance for
1292                  * this device and start the probe
1293                  * process.
1294                  */
1295                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1296                                           adacleanup, adastart,
1297                                           "ada", CAM_PERIPH_BIO,
1298                                           path, adaasync,
1299                                           AC_FOUND_DEVICE, cgd);
1300
1301                 if (status != CAM_REQ_CMP
1302                  && status != CAM_REQ_INPROG)
1303                         printf("adaasync: Unable to attach to new device "
1304                                 "due to status 0x%x\n", status);
1305                 break;
1306         }
1307         case AC_GETDEV_CHANGED:
1308         {
1309                 softc = (struct ada_softc *)periph->softc;
1310                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1311                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1312                 xpt_action((union ccb *)&cgd);
1313
1314                 /*
1315                  * Update our information based on the new Identify data.
1316                  */
1317                 adasetflags(softc, &cgd);
1318                 adasetgeom(softc, &cgd);
1319                 disk_resize(softc->disk, M_NOWAIT);
1320
1321                 cam_periph_async(periph, code, path, arg);
1322                 break;
1323         }
1324         case AC_ADVINFO_CHANGED:
1325         {
1326                 uintptr_t buftype;
1327
1328                 buftype = (uintptr_t)arg;
1329                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1330                         struct ada_softc *softc;
1331
1332                         softc = periph->softc;
1333                         disk_attr_changed(softc->disk, "GEOM::physpath",
1334                                           M_NOWAIT);
1335                 }
1336                 break;
1337         }
1338         case AC_SENT_BDR:
1339         case AC_BUS_RESET:
1340         {
1341                 softc = (struct ada_softc *)periph->softc;
1342                 cam_periph_async(periph, code, path, arg);
1343                 if (softc->state != ADA_STATE_NORMAL)
1344                         break;
1345                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1346                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1347                 xpt_action((union ccb *)&cgd);
1348                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1349                         softc->state = ADA_STATE_RAHEAD;
1350                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1351                         softc->state = ADA_STATE_WCACHE;
1352                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1353                       && (softc->zone_mode != ADA_ZONE_NONE))
1354                         softc->state = ADA_STATE_LOGDIR;
1355                 else
1356                     break;
1357                 if (cam_periph_acquire(periph) != 0)
1358                         softc->state = ADA_STATE_NORMAL;
1359                 else
1360                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1361         }
1362         default:
1363                 cam_periph_async(periph, code, path, arg);
1364                 break;
1365         }
1366 }
1367
1368 static int
1369 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1370 {
1371         char tmpbuf[40];
1372         struct ada_softc *softc;
1373         int error;
1374
1375         softc = (struct ada_softc *)arg1;
1376
1377         switch (softc->zone_mode) {
1378         case ADA_ZONE_DRIVE_MANAGED:
1379                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1380                 break;
1381         case ADA_ZONE_HOST_AWARE:
1382                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1383                 break;
1384         case ADA_ZONE_HOST_MANAGED:
1385                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1386                 break;
1387         case ADA_ZONE_NONE:
1388         default:
1389                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1390                 break;
1391         }
1392
1393         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1394
1395         return (error);
1396 }
1397
1398 static int
1399 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1400 {
1401         char tmpbuf[180];
1402         struct ada_softc *softc;
1403         struct sbuf sb;
1404         int error, first;
1405         unsigned int i;
1406
1407         softc = (struct ada_softc *)arg1;
1408
1409         error = 0;
1410         first = 1;
1411         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1412
1413         for (i = 0; i < sizeof(ada_zone_desc_table) /
1414              sizeof(ada_zone_desc_table[0]); i++) {
1415                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1416                         if (first == 0)
1417                                 sbuf_printf(&sb, ", ");
1418                         else
1419                                 first = 0;
1420                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1421                 }
1422         }
1423
1424         if (first == 1)
1425                 sbuf_printf(&sb, "None");
1426
1427         sbuf_finish(&sb);
1428
1429         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1430
1431         return (error);
1432 }
1433
1434
1435 static void
1436 adasysctlinit(void *context, int pending)
1437 {
1438         struct cam_periph *periph;
1439         struct ada_softc *softc;
1440         char tmpstr[32], tmpstr2[16];
1441
1442         periph = (struct cam_periph *)context;
1443
1444         /* periph was held for us when this task was enqueued */
1445         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1446                 cam_periph_release(periph);
1447                 return;
1448         }
1449
1450         softc = (struct ada_softc *)periph->softc;
1451         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1452         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1453
1454         sysctl_ctx_init(&softc->sysctl_ctx);
1455         softc->flags |= ADA_FLAG_SCTX_INIT;
1456         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1457                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1458                 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, tmpstr, "device_index");
1459         if (softc->sysctl_tree == NULL) {
1460                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1461                 cam_periph_release(periph);
1462                 return;
1463         }
1464
1465         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1466                 OID_AUTO, "delete_method",
1467                 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
1468                 softc, 0, adadeletemethodsysctl, "A",
1469                 "BIO_DELETE execution method");
1470         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1471                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1472                 "trim_count", CTLFLAG_RD, &softc->trim_count,
1473                 "Total number of dsm commands sent");
1474         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1475                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1476                 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges,
1477                 "Total number of ranges in dsm commands");
1478         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1479                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1480                 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas,
1481                 "Total lbas in the dsm commands sent");
1482         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1483                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1484                 &softc->read_ahead, 0, "Enable disk read ahead.");
1485         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1486                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1487                 &softc->write_cache, 0, "Enable disk write cache.");
1488         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1489                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1490                 &softc->unmappedio, 0, "Unmapped I/O leaf");
1491         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1492                 OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1493                 &softc->rotating, 0, "Rotating media");
1494         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1495                 OID_AUTO, "zone_mode",
1496                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
1497                 softc, 0, adazonemodesysctl, "A",
1498                 "Zone Mode");
1499         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1500                 OID_AUTO, "zone_support",
1501                 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
1502                 softc, 0, adazonesupsysctl, "A",
1503                 "Zone Support");
1504         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1505                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1506                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1507                 "Optimal Number of Open Sequential Write Preferred Zones");
1508         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1509                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1510                 "optimal_nonseq_zones", CTLFLAG_RD,
1511                 &softc->optimal_nonseq_zones,
1512                 "Optimal Number of Non-Sequentially Written Sequential Write "
1513                 "Preferred Zones");
1514         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1515                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1516                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1517                 "Maximum Number of Open Sequential Write Required Zones");
1518
1519 #ifdef CAM_TEST_FAILURE
1520         /*
1521          * Add a 'door bell' sysctl which allows one to set it from userland
1522          * and cause something bad to happen.  For the moment, we only allow
1523          * whacking the next read or write.
1524          */
1525         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1526                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1527                 &softc->force_read_error, 0,
1528                 "Force a read error for the next N reads.");
1529         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1530                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1531                 &softc->force_write_error, 0,
1532                 "Force a write error for the next N writes.");
1533         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1534                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1535                 &softc->periodic_read_error, 0,
1536                 "Force a read error every N reads (don't set too low).");
1537         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1538                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1539                 periph, 0, cam_periph_invalidate_sysctl, "I",
1540                 "Write 1 to invalidate the drive immediately");
1541 #endif
1542
1543 #ifdef CAM_IO_STATS
1544         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1545                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1546                 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "Statistics");
1547         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1548                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1549                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1550                 &softc->timeouts, 0,
1551                 "Device timeouts reported by the SIM");
1552         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1553                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1554                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1555                 &softc->errors, 0,
1556                 "Transport errors reported by the SIM.");
1557         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1558                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1559                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1560                 &softc->invalidations, 0,
1561                 "Device pack invalidations.");
1562 #endif
1563
1564         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1565             softc->sysctl_tree);
1566
1567         cam_periph_release(periph);
1568 }
1569
1570 static int
1571 adagetattr(struct bio *bp)
1572 {
1573         int ret;
1574         struct cam_periph *periph;
1575
1576         if (g_handleattr_int(bp, "GEOM::canspeedup", ada_enable_biospeedup))
1577                 return (EJUSTRETURN);
1578
1579         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1580         cam_periph_lock(periph);
1581         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1582             periph->path);
1583         cam_periph_unlock(periph);
1584         if (ret == 0)
1585                 bp->bio_completed = bp->bio_length;
1586         return ret;
1587 }
1588
1589 static int
1590 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1591 {
1592         char buf[16];
1593         const char *p;
1594         struct ada_softc *softc;
1595         int i, error, value, methods;
1596
1597         softc = (struct ada_softc *)arg1;
1598
1599         value = softc->delete_method;
1600         if (value < 0 || value > ADA_DELETE_MAX)
1601                 p = "UNKNOWN";
1602         else
1603                 p = ada_delete_method_names[value];
1604         strncpy(buf, p, sizeof(buf));
1605         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1606         if (error != 0 || req->newptr == NULL)
1607                 return (error);
1608         methods = 1 << ADA_DELETE_DISABLE;
1609         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1610             !(softc->flags & ADA_FLAG_CAN_48BIT))
1611                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1612         if (softc->flags & ADA_FLAG_CAN_TRIM)
1613                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1614         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1615                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1616         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1617                 if (!(methods & (1 << i)) ||
1618                     strcmp(buf, ada_delete_method_names[i]) != 0)
1619                         continue;
1620                 softc->delete_method = i;
1621                 return (0);
1622         }
1623         return (EINVAL);
1624 }
1625
1626 static void
1627 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1628 {
1629         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1630             (cgd->inq_flags & SID_DMA))
1631                 softc->flags |= ADA_FLAG_CAN_DMA;
1632         else
1633                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1634
1635         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1636                 softc->flags |= ADA_FLAG_CAN_48BIT;
1637                 if (cgd->inq_flags & SID_DMA48)
1638                         softc->flags |= ADA_FLAG_CAN_DMA48;
1639                 else
1640                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1641         } else
1642                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1643
1644         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1645                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1646         else
1647                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1648
1649         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1650                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1651         else
1652                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1653
1654         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1655             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1656                 softc->flags |= ADA_FLAG_CAN_NCQ;
1657         else
1658                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1659
1660         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1661             (cgd->inq_flags & SID_DMA) &&
1662             (softc->quirks & ADA_Q_NO_TRIM) == 0) {
1663                 softc->flags |= ADA_FLAG_CAN_TRIM;
1664                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1665                 if (cgd->ident_data.max_dsm_blocks != 0) {
1666                         softc->trim_max_ranges =
1667                             min(cgd->ident_data.max_dsm_blocks *
1668                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1669                 }
1670                 /*
1671                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1672                  * to do NCQ trims, if we support trims at all. We also need
1673                  * support from the SIM to do things properly. Perhaps we
1674                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1675                  * set too...
1676                  */
1677                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1678                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1679                     (cgd->ident_data.satacapabilities2 &
1680                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1681                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1682                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1683                 else
1684                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1685         } else
1686                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1687
1688         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1689                 softc->flags |= ADA_FLAG_CAN_CFA;
1690         else
1691                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1692
1693         /*
1694          * Now that we've set the appropriate flags, setup the delete
1695          * method.
1696          */
1697         adasetdeletemethod(softc);
1698
1699         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1700          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1701                 softc->flags |= ADA_FLAG_CAN_LOG;
1702         else
1703                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1704
1705         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1706              ATA_SUPPORT_ZONE_HOST_AWARE)
1707                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1708         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1709                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1710               || (softc->quirks & ADA_Q_SMR_DM))
1711                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1712         else
1713                 softc->zone_mode = ADA_ZONE_NONE;
1714
1715         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1716                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1717         else
1718                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1719
1720         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1721                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1722         else
1723                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1724 }
1725
1726 static cam_status
1727 adaregister(struct cam_periph *periph, void *arg)
1728 {
1729         struct ada_softc *softc;
1730         struct ccb_getdev *cgd;
1731         struct disk_params *dp;
1732         struct sbuf sb;
1733         char   *announce_buf;
1734         caddr_t match;
1735         int quirks;
1736
1737         cgd = (struct ccb_getdev *)arg;
1738         if (cgd == NULL) {
1739                 printf("adaregister: no getdev CCB, can't register device\n");
1740                 return(CAM_REQ_CMP_ERR);
1741         }
1742
1743         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1744             M_NOWAIT|M_ZERO);
1745
1746         if (softc == NULL) {
1747                 printf("adaregister: Unable to probe new device. "
1748                     "Unable to allocate softc\n");
1749                 return(CAM_REQ_CMP_ERR);
1750         }
1751
1752         announce_buf = softc->announce_temp;
1753         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1754
1755         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1756                 printf("adaregister: Unable to probe new device. "
1757                        "Unable to allocate iosched memory\n");
1758                 free(softc, M_DEVBUF);
1759                 return(CAM_REQ_CMP_ERR);
1760         }
1761
1762         periph->softc = softc;
1763         xpt_path_inq(&softc->cpi, periph->path);
1764
1765         /*
1766          * See if this device has any quirks.
1767          */
1768         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1769                                (caddr_t)ada_quirk_table,
1770                                nitems(ada_quirk_table),
1771                                sizeof(*ada_quirk_table), ata_identify_match);
1772         if (match != NULL)
1773                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1774         else
1775                 softc->quirks = ADA_Q_NONE;
1776
1777         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1778
1779         /*
1780          * Register this media as a disk
1781          */
1782         (void)cam_periph_hold(periph, PRIBIO);
1783         cam_periph_unlock(periph);
1784         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1785             "kern.cam.ada.%d.quirks", periph->unit_number);
1786         quirks = softc->quirks;
1787         TUNABLE_INT_FETCH(announce_buf, &quirks);
1788         softc->quirks = quirks;
1789         softc->read_ahead = -1;
1790         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1791             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1792         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1793         softc->write_cache = -1;
1794         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1795             "kern.cam.ada.%d.write_cache", periph->unit_number);
1796         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1797
1798         /*
1799          * Set support flags based on the Identify data and quirks.
1800          */
1801         adasetflags(softc, cgd);
1802         if (softc->cpi.hba_misc & PIM_ATA_EXT)
1803                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1804
1805         /* Disable queue sorting for non-rotational media by default. */
1806         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1807                 softc->rotating = 0;
1808         } else {
1809                 softc->rotating = 1;
1810         }
1811         cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1812         softc->disk = disk_alloc();
1813         adasetgeom(softc, cgd);
1814         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1815                           periph->unit_number, softc->params.secsize,
1816                           DEVSTAT_ALL_SUPPORTED,
1817                           DEVSTAT_TYPE_DIRECT |
1818                           XPORT_DEVSTAT_TYPE(softc->cpi.transport),
1819                           DEVSTAT_PRIORITY_DISK);
1820         softc->disk->d_open = adaopen;
1821         softc->disk->d_close = adaclose;
1822         softc->disk->d_strategy = adastrategy;
1823         softc->disk->d_getattr = adagetattr;
1824         softc->disk->d_dump = adadump;
1825         softc->disk->d_gone = adadiskgonecb;
1826         softc->disk->d_name = "ada";
1827         softc->disk->d_drv1 = periph;
1828         softc->disk->d_unit = periph->unit_number;
1829
1830         /*
1831          * Acquire a reference to the periph before we register with GEOM.
1832          * We'll release this reference once GEOM calls us back (via
1833          * adadiskgonecb()) telling us that our provider has been freed.
1834          */
1835         if (cam_periph_acquire(periph) != 0) {
1836                 xpt_print(periph->path, "%s: lost periph during "
1837                           "registration!\n", __func__);
1838                 cam_periph_lock(periph);
1839                 return (CAM_REQ_CMP_ERR);
1840         }
1841         disk_create(softc->disk, DISK_VERSION);
1842         cam_periph_lock(periph);
1843
1844         dp = &softc->params;
1845         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1846             "%juMB (%ju %u byte sectors)",
1847             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1848             (uintmax_t)dp->sectors, dp->secsize);
1849
1850         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1851         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1852         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1853         sbuf_finish(&sb);
1854         sbuf_putbuf(&sb);
1855
1856         /*
1857          * Create our sysctl variables, now that we know
1858          * we have successfully attached.
1859          */
1860         if (cam_periph_acquire(periph) == 0)
1861                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1862
1863         /*
1864          * Add async callbacks for bus reset and
1865          * bus device reset calls.  I don't bother
1866          * checking if this fails as, in most cases,
1867          * the system will function just fine without
1868          * them and the only alternative would be to
1869          * not attach the device on failure.
1870          */
1871         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1872             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1873             adaasync, periph, periph->path);
1874
1875         /*
1876          * Schedule a periodic event to occasionally send an
1877          * ordered tag to a device.
1878          */
1879         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1880         callout_reset(&softc->sendordered_c,
1881             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1882             adasendorderedtag, softc);
1883
1884         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1885                 softc->state = ADA_STATE_RAHEAD;
1886         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1887                 softc->state = ADA_STATE_WCACHE;
1888         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1889                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1890                 softc->state = ADA_STATE_LOGDIR;
1891         } else {
1892                 /*
1893                  * Nothing to probe, so we can just transition to the
1894                  * normal state.
1895                  */
1896                 adaprobedone(periph, NULL);
1897                 return(CAM_REQ_CMP);
1898         }
1899
1900         xpt_schedule(periph, CAM_PRIORITY_DEV);
1901
1902         return(CAM_REQ_CMP);
1903 }
1904
1905 static int
1906 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1907 {
1908         uint64_t lastlba = (uint64_t)-1, lbas = 0;
1909         int c, lastcount = 0, off, ranges = 0;
1910
1911         bzero(req, sizeof(*req));
1912         TAILQ_INIT(&req->bps);
1913         do {
1914                 uint64_t lba = bp->bio_pblkno;
1915                 int count = bp->bio_bcount / softc->params.secsize;
1916
1917                 /* Try to extend the previous range. */
1918                 if (lba == lastlba) {
1919                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1920                         lastcount += c;
1921                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1922                         req->data[off + 6] = lastcount & 0xff;
1923                         req->data[off + 7] =
1924                                 (lastcount >> 8) & 0xff;
1925                         count -= c;
1926                         lba += c;
1927                         lbas += c;
1928                 }
1929
1930                 while (count > 0) {
1931                         c = min(count, ATA_DSM_RANGE_MAX);
1932                         off = ranges * ATA_DSM_RANGE_SIZE;
1933                         req->data[off + 0] = lba & 0xff;
1934                         req->data[off + 1] = (lba >> 8) & 0xff;
1935                         req->data[off + 2] = (lba >> 16) & 0xff;
1936                         req->data[off + 3] = (lba >> 24) & 0xff;
1937                         req->data[off + 4] = (lba >> 32) & 0xff;
1938                         req->data[off + 5] = (lba >> 40) & 0xff;
1939                         req->data[off + 6] = c & 0xff;
1940                         req->data[off + 7] = (c >> 8) & 0xff;
1941                         lba += c;
1942                         lbas += c;
1943                         count -= c;
1944                         lastcount = c;
1945                         ranges++;
1946                         /*
1947                          * Its the caller's responsibility to ensure the
1948                          * request will fit so we don't need to check for
1949                          * overrun here
1950                          */
1951                 }
1952                 lastlba = lba;
1953                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1954
1955                 bp = cam_iosched_next_trim(softc->cam_iosched);
1956                 if (bp == NULL)
1957                         break;
1958                 if (bp->bio_bcount / softc->params.secsize >
1959                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
1960                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
1961                         break;
1962                 }
1963         } while (1);
1964         softc->trim_count++;
1965         softc->trim_ranges += ranges;
1966         softc->trim_lbas += lbas;
1967
1968         return (ranges);
1969 }
1970
1971 static void
1972 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1973 {
1974         struct trim_request *req = &softc->trim_req;
1975         int ranges;
1976
1977         ranges = ada_dsmtrim_req_create(softc, bp, req);
1978         cam_fill_ataio(ataio,
1979             ada_retry_count,
1980             adadone,
1981             CAM_DIR_OUT,
1982             0,
1983             req->data,
1984             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1985             ada_default_timeout * 1000);
1986         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1987             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
1988 }
1989
1990 static void
1991 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1992 {
1993         struct trim_request *req = &softc->trim_req;
1994         int ranges;
1995
1996         ranges = ada_dsmtrim_req_create(softc, bp, req);
1997         cam_fill_ataio(ataio,
1998             ada_retry_count,
1999             adadone,
2000             CAM_DIR_OUT,
2001             0,
2002             req->data,
2003             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2004             ada_default_timeout * 1000);
2005         ata_ncq_cmd(ataio,
2006             ATA_SEND_FPDMA_QUEUED,
2007             0,
2008             howmany(ranges, ATA_DSM_BLK_RANGES));
2009         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2010         ataio->ata_flags |= ATA_FLAG_AUX;
2011         ataio->aux = 1;
2012 }
2013
2014 static void
2015 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2016 {
2017         struct trim_request *req = &softc->trim_req;
2018         uint64_t lba = bp->bio_pblkno;
2019         uint16_t count = bp->bio_bcount / softc->params.secsize;
2020
2021         bzero(req, sizeof(*req));
2022         TAILQ_INIT(&req->bps);
2023         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2024
2025         cam_fill_ataio(ataio,
2026             ada_retry_count,
2027             adadone,
2028             CAM_DIR_NONE,
2029             0,
2030             NULL,
2031             0,
2032             ada_default_timeout*1000);
2033
2034         if (count >= 256)
2035                 count = 0;
2036         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2037 }
2038
2039 static int
2040 ada_zone_bio_to_ata(int disk_zone_cmd)
2041 {
2042         switch (disk_zone_cmd) {
2043         case DISK_ZONE_OPEN:
2044                 return ATA_ZM_OPEN_ZONE;
2045         case DISK_ZONE_CLOSE:
2046                 return ATA_ZM_CLOSE_ZONE;
2047         case DISK_ZONE_FINISH:
2048                 return ATA_ZM_FINISH_ZONE;
2049         case DISK_ZONE_RWP:
2050                 return ATA_ZM_RWP;
2051         }
2052
2053         return -1;
2054 }
2055
2056 static int
2057 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2058              int *queue_ccb)
2059 {
2060         struct ada_softc *softc;
2061         int error;
2062
2063         error = 0;
2064
2065         if (bp->bio_cmd != BIO_ZONE) {
2066                 error = EINVAL;
2067                 goto bailout;
2068         }
2069
2070         softc = periph->softc;
2071
2072         switch (bp->bio_zone.zone_cmd) {
2073         case DISK_ZONE_OPEN:
2074         case DISK_ZONE_CLOSE:
2075         case DISK_ZONE_FINISH:
2076         case DISK_ZONE_RWP: {
2077                 int zone_flags;
2078                 int zone_sa;
2079                 uint64_t lba;
2080
2081                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2082                 if (zone_sa == -1) {
2083                         xpt_print(periph->path, "Cannot translate zone "
2084                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2085                         error = EINVAL;
2086                         goto bailout;
2087                 }
2088
2089                 zone_flags = 0;
2090                 lba = bp->bio_zone.zone_params.rwp.id;
2091
2092                 if (bp->bio_zone.zone_params.rwp.flags &
2093                     DISK_ZONE_RWP_FLAG_ALL)
2094                         zone_flags |= ZBC_OUT_ALL;
2095
2096                 ata_zac_mgmt_out(&ccb->ataio,
2097                                  /*retries*/ ada_retry_count,
2098                                  /*cbfcnp*/ adadone,
2099                                  /*use_ncq*/ (softc->flags &
2100                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2101                                  /*zm_action*/ zone_sa,
2102                                  /*zone_id*/ lba,
2103                                  /*zone_flags*/ zone_flags,
2104                                  /*sector_count*/ 0,
2105                                  /*data_ptr*/ NULL,
2106                                  /*dxfer_len*/ 0,
2107                                  /*timeout*/ ada_default_timeout * 1000);
2108                 *queue_ccb = 1;
2109
2110                 break;
2111         }
2112         case DISK_ZONE_REPORT_ZONES: {
2113                 uint8_t *rz_ptr;
2114                 uint32_t num_entries, alloc_size;
2115                 struct disk_zone_report *rep;
2116
2117                 rep = &bp->bio_zone.zone_params.report;
2118
2119                 num_entries = rep->entries_allocated;
2120                 if (num_entries == 0) {
2121                         xpt_print(periph->path, "No entries allocated for "
2122                             "Report Zones request\n");
2123                         error = EINVAL;
2124                         goto bailout;
2125                 }
2126                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2127                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2128                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2129                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2130                 if (rz_ptr == NULL) {
2131                         xpt_print(periph->path, "Unable to allocate memory "
2132                            "for Report Zones request\n");
2133                         error = ENOMEM;
2134                         goto bailout;
2135                 }
2136
2137                 ata_zac_mgmt_in(&ccb->ataio,
2138                                 /*retries*/ ada_retry_count,
2139                                 /*cbcfnp*/ adadone,
2140                                 /*use_ncq*/ (softc->flags &
2141                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2142                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2143                                 /*zone_id*/ rep->starting_id,
2144                                 /*zone_flags*/ rep->rep_options,
2145                                 /*data_ptr*/ rz_ptr,
2146                                 /*dxfer_len*/ alloc_size,
2147                                 /*timeout*/ ada_default_timeout * 1000);
2148
2149                 /*
2150                  * For BIO_ZONE, this isn't normally needed.  However, it
2151                  * is used by devstat_end_transaction_bio() to determine
2152                  * how much data was transferred.
2153                  */
2154                 /*
2155                  * XXX KDM we have a problem.  But I'm not sure how to fix
2156                  * it.  devstat uses bio_bcount - bio_resid to calculate
2157                  * the amount of data transferred.   The GEOM disk code
2158                  * uses bio_length - bio_resid to calculate the amount of
2159                  * data in bio_completed.  We have different structure
2160                  * sizes above and below the ada(4) driver.  So, if we
2161                  * use the sizes above, the amount transferred won't be
2162                  * quite accurate for devstat.  If we use different sizes
2163                  * for bio_bcount and bio_length (above and below
2164                  * respectively), then the residual needs to match one or
2165                  * the other.  Everything is calculated after the bio
2166                  * leaves the driver, so changing the values around isn't
2167                  * really an option.  For now, just set the count to the
2168                  * passed in length.  This means that the calculations
2169                  * above (e.g. bio_completed) will be correct, but the
2170                  * amount of data reported to devstat will be slightly
2171                  * under or overstated.
2172                  */
2173                 bp->bio_bcount = bp->bio_length;
2174
2175                 *queue_ccb = 1;
2176
2177                 break;
2178         }
2179         case DISK_ZONE_GET_PARAMS: {
2180                 struct disk_zone_disk_params *params;
2181
2182                 params = &bp->bio_zone.zone_params.disk_params;
2183                 bzero(params, sizeof(*params));
2184
2185                 switch (softc->zone_mode) {
2186                 case ADA_ZONE_DRIVE_MANAGED:
2187                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2188                         break;
2189                 case ADA_ZONE_HOST_AWARE:
2190                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2191                         break;
2192                 case ADA_ZONE_HOST_MANAGED:
2193                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2194                         break;
2195                 default:
2196                 case ADA_ZONE_NONE:
2197                         params->zone_mode = DISK_ZONE_MODE_NONE;
2198                         break;
2199                 }
2200
2201                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2202                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2203
2204                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2205                         params->optimal_seq_zones = softc->optimal_seq_zones;
2206                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2207                 }
2208
2209                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2210                         params->optimal_nonseq_zones =
2211                             softc->optimal_nonseq_zones;
2212                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2213                 }
2214
2215                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2216                         params->max_seq_zones = softc->max_seq_zones;
2217                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2218                 }
2219                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2220                         params->flags |= DISK_ZONE_RZ_SUP;
2221
2222                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2223                         params->flags |= DISK_ZONE_OPEN_SUP;
2224
2225                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2226                         params->flags |= DISK_ZONE_CLOSE_SUP;
2227
2228                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2229                         params->flags |= DISK_ZONE_FINISH_SUP;
2230
2231                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2232                         params->flags |= DISK_ZONE_RWP_SUP;
2233                 break;
2234         }
2235         default:
2236                 break;
2237         }
2238 bailout:
2239         return (error);
2240 }
2241
2242 static void
2243 adastart(struct cam_periph *periph, union ccb *start_ccb)
2244 {
2245         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2246         struct ccb_ataio *ataio = &start_ccb->ataio;
2247
2248         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2249
2250         switch (softc->state) {
2251         case ADA_STATE_NORMAL:
2252         {
2253                 struct bio *bp;
2254                 u_int8_t tag_code;
2255
2256                 bp = cam_iosched_next_bio(softc->cam_iosched);
2257                 if (bp == NULL) {
2258                         xpt_release_ccb(start_ccb);
2259                         break;
2260                 }
2261
2262                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2263                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2264                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2265                         softc->flags |= ADA_FLAG_WAS_OTAG;
2266                         tag_code = 0;
2267                 } else {
2268                         tag_code = 1;
2269                 }
2270                 switch (bp->bio_cmd) {
2271                 case BIO_WRITE:
2272                 case BIO_READ:
2273                 {
2274                         uint64_t lba = bp->bio_pblkno;
2275                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2276                         void *data_ptr;
2277                         int rw_op;
2278
2279                         if (bp->bio_cmd == BIO_WRITE) {
2280                                 softc->flags |= ADA_FLAG_DIRTY;
2281                                 rw_op = CAM_DIR_OUT;
2282                         } else {
2283                                 rw_op = CAM_DIR_IN;
2284                         }
2285
2286                         data_ptr = bp->bio_data;
2287                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2288                                 rw_op |= CAM_DATA_BIO;
2289                                 data_ptr = bp;
2290                         }
2291
2292 #ifdef CAM_TEST_FAILURE
2293                         int fail = 0;
2294
2295                         /*
2296                          * Support the failure ioctls.  If the command is a
2297                          * read, and there are pending forced read errors, or
2298                          * if a write and pending write errors, then fail this
2299                          * operation with EIO.  This is useful for testing
2300                          * purposes.  Also, support having every Nth read fail.
2301                          *
2302                          * This is a rather blunt tool.
2303                          */
2304                         if (bp->bio_cmd == BIO_READ) {
2305                                 if (softc->force_read_error) {
2306                                         softc->force_read_error--;
2307                                         fail = 1;
2308                                 }
2309                                 if (softc->periodic_read_error > 0) {
2310                                         if (++softc->periodic_read_count >=
2311                                             softc->periodic_read_error) {
2312                                                 softc->periodic_read_count = 0;
2313                                                 fail = 1;
2314                                         }
2315                                 }
2316                         } else {
2317                                 if (softc->force_write_error) {
2318                                         softc->force_write_error--;
2319                                         fail = 1;
2320                                 }
2321                         }
2322                         if (fail) {
2323                                 biofinish(bp, NULL, EIO);
2324                                 xpt_release_ccb(start_ccb);
2325                                 adaschedule(periph);
2326                                 return;
2327                         }
2328 #endif
2329                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2330                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2331                             PAGE_SIZE == bp->bio_ma_n,
2332                             ("Short bio %p", bp));
2333                         cam_fill_ataio(ataio,
2334                             ada_retry_count,
2335                             adadone,
2336                             rw_op,
2337                             0,
2338                             data_ptr,
2339                             bp->bio_bcount,
2340                             ada_default_timeout*1000);
2341
2342                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2343                                 if (bp->bio_cmd == BIO_READ) {
2344                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2345                                             lba, count);
2346                                 } else {
2347                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2348                                             lba, count);
2349                                 }
2350                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2351                             (lba + count >= ATA_MAX_28BIT_LBA ||
2352                             count > 256)) {
2353                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2354                                         if (bp->bio_cmd == BIO_READ) {
2355                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2356                                                     0, lba, count);
2357                                         } else {
2358                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2359                                                     0, lba, count);
2360                                         }
2361                                 } else {
2362                                         if (bp->bio_cmd == BIO_READ) {
2363                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2364                                                     0, lba, count);
2365                                         } else {
2366                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2367                                                     0, lba, count);
2368                                         }
2369                                 }
2370                         } else {
2371                                 if (count == 256)
2372                                         count = 0;
2373                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2374                                         if (bp->bio_cmd == BIO_READ) {
2375                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2376                                                     0, lba, count);
2377                                         } else {
2378                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2379                                                     0, lba, count);
2380                                         }
2381                                 } else {
2382                                         if (bp->bio_cmd == BIO_READ) {
2383                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2384                                                     0, lba, count);
2385                                         } else {
2386                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2387                                                     0, lba, count);
2388                                         }
2389                                 }
2390                         }
2391                         break;
2392                 }
2393                 case BIO_DELETE:
2394                         switch (softc->delete_method) {
2395                         case ADA_DELETE_NCQ_DSM_TRIM:
2396                                 ada_ncq_dsmtrim(softc, bp, ataio);
2397                                 break;
2398                         case ADA_DELETE_DSM_TRIM:
2399                                 ada_dsmtrim(softc, bp, ataio);
2400                                 break;
2401                         case ADA_DELETE_CFA_ERASE:
2402                                 ada_cfaerase(softc, bp, ataio);
2403                                 break;
2404                         default:
2405                                 biofinish(bp, NULL, EOPNOTSUPP);
2406                                 xpt_release_ccb(start_ccb);
2407                                 adaschedule(periph);
2408                                 return;
2409                         }
2410                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2411                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2412                         cam_iosched_submit_trim(softc->cam_iosched);
2413                         goto out;
2414                 case BIO_FLUSH:
2415                         cam_fill_ataio(ataio,
2416                             1,
2417                             adadone,
2418                             CAM_DIR_NONE,
2419                             0,
2420                             NULL,
2421                             0,
2422                             ada_default_timeout*1000);
2423
2424                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2425                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2426                         else
2427                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2428                         break;
2429                 case BIO_ZONE: {
2430                         int error, queue_ccb;
2431
2432                         queue_ccb = 0;
2433
2434                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2435                         if ((error != 0)
2436                          || (queue_ccb == 0)) {
2437                                 biofinish(bp, NULL, error);
2438                                 xpt_release_ccb(start_ccb);
2439                                 return;
2440                         }
2441                         break;
2442                 }
2443                 default:
2444                         biofinish(bp, NULL, EOPNOTSUPP);
2445                         xpt_release_ccb(start_ccb);
2446                         return;
2447                 }
2448                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2449                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2450 out:
2451                 start_ccb->ccb_h.ccb_bp = bp;
2452                 softc->outstanding_cmds++;
2453                 softc->refcount++;
2454                 cam_periph_unlock(periph);
2455                 xpt_action(start_ccb);
2456                 cam_periph_lock(periph);
2457
2458                 /* May have more work to do, so ensure we stay scheduled */
2459                 adaschedule(periph);
2460                 break;
2461         }
2462         case ADA_STATE_RAHEAD:
2463         case ADA_STATE_WCACHE:
2464         {
2465                 cam_fill_ataio(ataio,
2466                     1,
2467                     adadone,
2468                     CAM_DIR_NONE,
2469                     0,
2470                     NULL,
2471                     0,
2472                     ada_default_timeout*1000);
2473
2474                 if (softc->state == ADA_STATE_RAHEAD) {
2475                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2476                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2477                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2478                 } else {
2479                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2480                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2481                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2482                 }
2483                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2484                 xpt_action(start_ccb);
2485                 break;
2486         }
2487         case ADA_STATE_LOGDIR:
2488         {
2489                 struct ata_gp_log_dir *log_dir;
2490
2491                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2492                         adaprobedone(periph, start_ccb);
2493                         break;
2494                 }
2495
2496                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2497                 if (log_dir == NULL) {
2498                         xpt_print(periph->path, "Couldn't malloc log_dir "
2499                             "data\n");
2500                         softc->state = ADA_STATE_NORMAL;
2501                         xpt_release_ccb(start_ccb);
2502                         break;
2503                 }
2504
2505
2506                 ata_read_log(ataio,
2507                     /*retries*/1,
2508                     /*cbfcnp*/adadone,
2509                     /*log_address*/ ATA_LOG_DIRECTORY,
2510                     /*page_number*/ 0,
2511                     /*block_count*/ 1,
2512                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2513                                  CAM_ATAIO_DMA : 0,
2514                     /*data_ptr*/ (uint8_t *)log_dir,
2515                     /*dxfer_len*/sizeof(*log_dir),
2516                     /*timeout*/ada_default_timeout*1000);
2517
2518                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2519                 xpt_action(start_ccb);
2520                 break;
2521         }
2522         case ADA_STATE_IDDIR:
2523         {
2524                 struct ata_identify_log_pages *id_dir;
2525
2526                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2527                 if (id_dir == NULL) {
2528                         xpt_print(periph->path, "Couldn't malloc id_dir "
2529                             "data\n");
2530                         adaprobedone(periph, start_ccb);
2531                         break;
2532                 }
2533
2534                 ata_read_log(ataio,
2535                     /*retries*/1,
2536                     /*cbfcnp*/adadone,
2537                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2538                     /*page_number*/ ATA_IDL_PAGE_LIST,
2539                     /*block_count*/ 1,
2540                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2541                                  CAM_ATAIO_DMA : 0,
2542                     /*data_ptr*/ (uint8_t *)id_dir,
2543                     /*dxfer_len*/ sizeof(*id_dir),
2544                     /*timeout*/ada_default_timeout*1000);
2545
2546                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2547                 xpt_action(start_ccb);
2548                 break;
2549         }
2550         case ADA_STATE_SUP_CAP:
2551         {
2552                 struct ata_identify_log_sup_cap *sup_cap;
2553
2554                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2555                 if (sup_cap == NULL) {
2556                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2557                             "data\n");
2558                         adaprobedone(periph, start_ccb);
2559                         break;
2560                 }
2561
2562                 ata_read_log(ataio,
2563                     /*retries*/1,
2564                     /*cbfcnp*/adadone,
2565                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2566                     /*page_number*/ ATA_IDL_SUP_CAP,
2567                     /*block_count*/ 1,
2568                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2569                                  CAM_ATAIO_DMA : 0,
2570                     /*data_ptr*/ (uint8_t *)sup_cap,
2571                     /*dxfer_len*/ sizeof(*sup_cap),
2572                     /*timeout*/ada_default_timeout*1000);
2573
2574                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2575                 xpt_action(start_ccb);
2576                 break;
2577         }
2578         case ADA_STATE_ZONE:
2579         {
2580                 struct ata_zoned_info_log *ata_zone;
2581
2582                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2583                 if (ata_zone == NULL) {
2584                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2585                             "data\n");
2586                         adaprobedone(periph, start_ccb);
2587                         break;
2588                 }
2589
2590                 ata_read_log(ataio,
2591                     /*retries*/1,
2592                     /*cbfcnp*/adadone,
2593                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2594                     /*page_number*/ ATA_IDL_ZDI,
2595                     /*block_count*/ 1,
2596                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2597                                  CAM_ATAIO_DMA : 0,
2598                     /*data_ptr*/ (uint8_t *)ata_zone,
2599                     /*dxfer_len*/ sizeof(*ata_zone),
2600                     /*timeout*/ada_default_timeout*1000);
2601
2602                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2603                 xpt_action(start_ccb);
2604                 break;
2605         }
2606         }
2607 }
2608
2609 static void
2610 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2611 {
2612         struct ada_softc *softc;
2613
2614         softc = (struct ada_softc *)periph->softc;
2615
2616         if (ccb != NULL)
2617                 xpt_release_ccb(ccb);
2618
2619         softc->state = ADA_STATE_NORMAL;
2620         softc->flags |= ADA_FLAG_PROBED;
2621         adaschedule(periph);
2622         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2623                 softc->flags |= ADA_FLAG_ANNOUNCED;
2624                 cam_periph_unhold(periph);
2625         } else {
2626                 cam_periph_release_locked(periph);
2627         }
2628 }
2629
2630 static void
2631 adazonedone(struct cam_periph *periph, union ccb *ccb)
2632 {
2633         struct bio *bp;
2634
2635         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2636
2637         switch (bp->bio_zone.zone_cmd) {
2638         case DISK_ZONE_OPEN:
2639         case DISK_ZONE_CLOSE:
2640         case DISK_ZONE_FINISH:
2641         case DISK_ZONE_RWP:
2642                 break;
2643         case DISK_ZONE_REPORT_ZONES: {
2644                 uint32_t avail_len;
2645                 struct disk_zone_report *rep;
2646                 struct scsi_report_zones_hdr *hdr;
2647                 struct scsi_report_zones_desc *desc;
2648                 struct disk_zone_rep_entry *entry;
2649                 uint32_t hdr_len, num_avail;
2650                 uint32_t num_to_fill, i;
2651
2652                 rep = &bp->bio_zone.zone_params.report;
2653                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2654                 /*
2655                  * Note that bio_resid isn't normally used for zone
2656                  * commands, but it is used by devstat_end_transaction_bio()
2657                  * to determine how much data was transferred.  Because
2658                  * the size of the SCSI/ATA data structures is different
2659                  * than the size of the BIO interface structures, the
2660                  * amount of data actually transferred from the drive will
2661                  * be different than the amount of data transferred to
2662                  * the user.
2663                  */
2664                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2665                 if (avail_len < sizeof(*hdr)) {
2666                         /*
2667                          * Is there a better error than EIO here?  We asked
2668                          * for at least the header, and we got less than
2669                          * that.
2670                          */
2671                         bp->bio_error = EIO;
2672                         bp->bio_flags |= BIO_ERROR;
2673                         bp->bio_resid = bp->bio_bcount;
2674                         break;
2675                 }
2676
2677                 hdr_len = le32dec(hdr->length);
2678                 if (hdr_len > 0)
2679                         rep->entries_available = hdr_len / sizeof(*desc);
2680                 else
2681                         rep->entries_available = 0;
2682                 /*
2683                  * NOTE: using the same values for the BIO version of the
2684                  * same field as the SCSI/ATA values.  This means we could
2685                  * get some additional values that aren't defined in bio.h
2686                  * if more values of the same field are defined later.
2687                  */
2688                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2689                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2690                 /*
2691                  * If the drive reports no entries that match the query,
2692                  * we're done.
2693                  */
2694                 if (hdr_len == 0) {
2695                         rep->entries_filled = 0;
2696                         bp->bio_resid = bp->bio_bcount;
2697                         break;
2698                 }
2699
2700                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2701                                 hdr_len / sizeof(*desc));
2702                 /*
2703                  * If the drive didn't return any data, then we're done.
2704                  */
2705                 if (num_avail == 0) {
2706                         rep->entries_filled = 0;
2707                         bp->bio_resid = bp->bio_bcount;
2708                         break;
2709                 }
2710
2711                 num_to_fill = min(num_avail, rep->entries_allocated);
2712                 /*
2713                  * If the user didn't allocate any entries for us to fill,
2714                  * we're done.
2715                  */
2716                 if (num_to_fill == 0) {
2717                         rep->entries_filled = 0;
2718                         bp->bio_resid = bp->bio_bcount;
2719                         break;
2720                 }
2721
2722                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2723                      i < num_to_fill; i++, desc++, entry++) {
2724                         /*
2725                          * NOTE: we're mapping the values here directly
2726                          * from the SCSI/ATA bit definitions to the bio.h
2727                          * definitions.  There is also a warning in
2728                          * disk_zone.h, but the impact is that if
2729                          * additional values are added in the SCSI/ATA
2730                          * specs these will be visible to consumers of
2731                          * this interface.
2732                          */
2733                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2734                         entry->zone_condition =
2735                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2736                             SRZ_ZONE_COND_SHIFT;
2737                         entry->zone_flags |= desc->zone_flags &
2738                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2739                         entry->zone_length = le64dec(desc->zone_length);
2740                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2741                         entry->write_pointer_lba =
2742                             le64dec(desc->write_pointer_lba);
2743                 }
2744                 rep->entries_filled = num_to_fill;
2745                 /*
2746                  * Note that this residual is accurate from the user's
2747                  * standpoint, but the amount transferred isn't accurate
2748                  * from the standpoint of what actually came back from the
2749                  * drive.
2750                  */
2751                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2752                 break;
2753         }
2754         case DISK_ZONE_GET_PARAMS:
2755         default:
2756                 /*
2757                  * In theory we should not get a GET_PARAMS bio, since it
2758                  * should be handled without queueing the command to the
2759                  * drive.
2760                  */
2761                 panic("%s: Invalid zone command %d", __func__,
2762                     bp->bio_zone.zone_cmd);
2763                 break;
2764         }
2765
2766         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2767                 free(ccb->ataio.data_ptr, M_ATADA);
2768 }
2769
2770
2771 static void
2772 adadone(struct cam_periph *periph, union ccb *done_ccb)
2773 {
2774         struct ada_softc *softc;
2775         struct ccb_ataio *ataio;
2776         struct cam_path *path;
2777         uint32_t priority;
2778         int state;
2779
2780         softc = (struct ada_softc *)periph->softc;
2781         ataio = &done_ccb->ataio;
2782         path = done_ccb->ccb_h.path;
2783         priority = done_ccb->ccb_h.pinfo.priority;
2784
2785         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2786
2787         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2788         switch (state) {
2789         case ADA_CCB_BUFFER_IO:
2790         case ADA_CCB_TRIM:
2791         {
2792                 struct bio *bp;
2793                 int error;
2794
2795                 cam_periph_lock(periph);
2796                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2797                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2798                         error = adaerror(done_ccb, 0, 0);
2799                         if (error == ERESTART) {
2800                                 /* A retry was scheduled, so just return. */
2801                                 cam_periph_unlock(periph);
2802                                 return;
2803                         }
2804                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2805                                 cam_release_devq(path,
2806                                                  /*relsim_flags*/0,
2807                                                  /*reduction*/0,
2808                                                  /*timeout*/0,
2809                                                  /*getcount_only*/0);
2810                         /*
2811                          * If we get an error on an NCQ DSM TRIM, fall back
2812                          * to a non-NCQ DSM TRIM forever. Please note that if
2813                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2814                          * However, for this one trim, we treat it as advisory
2815                          * and return success up the stack.
2816                          */
2817                         if (state == ADA_CCB_TRIM &&
2818                             error != 0 &&
2819                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2820                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2821                                 error = 0;
2822                                 adasetdeletemethod(softc);
2823                         }
2824                 } else {
2825                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2826                                 panic("REQ_CMP with QFRZN");
2827
2828                         error = 0;
2829                 }
2830                 bp->bio_error = error;
2831                 if (error != 0) {
2832                         bp->bio_resid = bp->bio_bcount;
2833                         bp->bio_flags |= BIO_ERROR;
2834                 } else {
2835                         if (bp->bio_cmd == BIO_ZONE)
2836                                 adazonedone(periph, done_ccb);
2837                         else if (state == ADA_CCB_TRIM)
2838                                 bp->bio_resid = 0;
2839                         else
2840                                 bp->bio_resid = ataio->resid;
2841
2842                         if ((bp->bio_resid > 0)
2843                          && (bp->bio_cmd != BIO_ZONE))
2844                                 bp->bio_flags |= BIO_ERROR;
2845                 }
2846                 softc->outstanding_cmds--;
2847                 if (softc->outstanding_cmds == 0)
2848                         softc->flags |= ADA_FLAG_WAS_OTAG;
2849
2850                 /*
2851                  * We need to call cam_iosched before we call biodone so that we
2852                  * don't measure any activity that happens in the completion
2853                  * routine, which in the case of sendfile can be quite
2854                  * extensive.  Release the periph refcount taken in adastart()
2855                  * for each CCB.
2856                  */
2857                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2858                 xpt_release_ccb(done_ccb);
2859                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2860                 softc->refcount--;
2861                 if (state == ADA_CCB_TRIM) {
2862                         TAILQ_HEAD(, bio) queue;
2863                         struct bio *bp1;
2864
2865                         TAILQ_INIT(&queue);
2866                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2867                         /*
2868                          * Normally, the xpt_release_ccb() above would make sure
2869                          * that when we have more work to do, that work would
2870                          * get kicked off. However, we specifically keep
2871                          * trim_running set to 0 before the call above to allow
2872                          * other I/O to progress when many BIO_DELETE requests
2873                          * are pushed down. We set trim_running to 0 and call
2874                          * daschedule again so that we don't stall if there are
2875                          * no other I/Os pending apart from BIO_DELETEs.
2876                          */
2877                         cam_iosched_trim_done(softc->cam_iosched);
2878                         adaschedule(periph);
2879                         cam_periph_unlock(periph);
2880                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2881                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2882                                 bp1->bio_error = error;
2883                                 if (error != 0) {
2884                                         bp1->bio_flags |= BIO_ERROR;
2885                                         bp1->bio_resid = bp1->bio_bcount;
2886                                 } else
2887                                         bp1->bio_resid = 0;
2888                                 biodone(bp1);
2889                         }
2890                 } else {
2891                         adaschedule(periph);
2892                         cam_periph_unlock(periph);
2893                         biodone(bp);
2894                 }
2895                 return;
2896         }
2897         case ADA_CCB_RAHEAD:
2898         {
2899                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2900                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2901                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2902                                 cam_release_devq(path, 0, 0, 0, FALSE);
2903                                 return;
2904                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2905                                 cam_release_devq(path,
2906                                     /*relsim_flags*/0,
2907                                     /*reduction*/0,
2908                                     /*timeout*/0,
2909                                     /*getcount_only*/0);
2910                         }
2911                 }
2912
2913                 /*
2914                  * Since our peripheral may be invalidated by an error
2915                  * above or an external event, we must release our CCB
2916                  * before releasing the reference on the peripheral.
2917                  * The peripheral will only go away once the last reference
2918                  * is removed, and we need it around for the CCB release
2919                  * operation.
2920                  */
2921
2922                 xpt_release_ccb(done_ccb);
2923                 softc->state = ADA_STATE_WCACHE;
2924                 xpt_schedule(periph, priority);
2925                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2926                 cam_release_devq(path, 0, 0, 0, FALSE);
2927                 return;
2928         }
2929         case ADA_CCB_WCACHE:
2930         {
2931                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2932                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2933                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2934                                 cam_release_devq(path, 0, 0, 0, FALSE);
2935                                 return;
2936                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2937                                 cam_release_devq(path,
2938                                     /*relsim_flags*/0,
2939                                     /*reduction*/0,
2940                                     /*timeout*/0,
2941                                     /*getcount_only*/0);
2942                         }
2943                 }
2944
2945                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2946                 cam_release_devq(path, 0, 0, 0, FALSE);
2947
2948                 if ((softc->flags & ADA_FLAG_CAN_LOG)
2949                  && (softc->zone_mode != ADA_ZONE_NONE)) {
2950                         xpt_release_ccb(done_ccb);
2951                         softc->state = ADA_STATE_LOGDIR;
2952                         xpt_schedule(periph, priority);
2953                 } else {
2954                         adaprobedone(periph, done_ccb);
2955                 }
2956                 return;
2957         }
2958         case ADA_CCB_LOGDIR:
2959         {
2960                 int error;
2961
2962                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2963                         error = 0;
2964                         softc->valid_logdir_len = 0;
2965                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
2966                         softc->valid_logdir_len =
2967                                 ataio->dxfer_len - ataio->resid;
2968                         if (softc->valid_logdir_len > 0)
2969                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
2970                                     min(softc->valid_logdir_len,
2971                                         sizeof(softc->ata_logdir)));
2972                         /*
2973                          * Figure out whether the Identify Device log is
2974                          * supported.  The General Purpose log directory
2975                          * has a header, and lists the number of pages
2976                          * available for each GP log identified by the
2977                          * offset into the list.
2978                          */
2979                         if ((softc->valid_logdir_len >=
2980                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
2981                          && (le16dec(softc->ata_logdir.header) ==
2982                              ATA_GP_LOG_DIR_VERSION)
2983                          && (le16dec(&softc->ata_logdir.num_pages[
2984                              (ATA_IDENTIFY_DATA_LOG *
2985                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
2986                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
2987                         } else {
2988                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
2989                         }
2990                 } else {
2991                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
2992                                          SF_RETRY_UA|SF_NO_PRINT);
2993                         if (error == ERESTART)
2994                                 return;
2995                         else if (error != 0) {
2996                                 /*
2997                                  * If we can't get the ATA log directory,
2998                                  * then ATA logs are effectively not
2999                                  * supported even if the bit is set in the
3000                                  * identify data.
3001                                  */
3002                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
3003                                                   ADA_FLAG_CAN_IDLOG);
3004                                 if ((done_ccb->ccb_h.status &
3005                                      CAM_DEV_QFRZN) != 0) {
3006                                         /* Don't wedge this device's queue */
3007                                         cam_release_devq(done_ccb->ccb_h.path,
3008                                                          /*relsim_flags*/0,
3009                                                          /*reduction*/0,
3010                                                          /*timeout*/0,
3011                                                          /*getcount_only*/0);
3012                                 }
3013                         }
3014
3015
3016                 }
3017
3018                 free(ataio->data_ptr, M_ATADA);
3019
3020                 if ((error == 0)
3021                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3022                         softc->state = ADA_STATE_IDDIR;
3023                         xpt_release_ccb(done_ccb);
3024                         xpt_schedule(periph, priority);
3025                 } else
3026                         adaprobedone(periph, done_ccb);
3027
3028                 return;
3029         }
3030         case ADA_CCB_IDDIR: {
3031                 int error;
3032
3033                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3034                         off_t entries_offset, max_entries;
3035                         error = 0;
3036
3037                         softc->valid_iddir_len = 0;
3038                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3039                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3040                                           ADA_FLAG_CAN_ZONE);
3041                         softc->valid_iddir_len =
3042                                 ataio->dxfer_len - ataio->resid;
3043                         if (softc->valid_iddir_len > 0)
3044                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3045                                     min(softc->valid_iddir_len,
3046                                         sizeof(softc->ata_iddir)));
3047
3048                         entries_offset =
3049                             __offsetof(struct ata_identify_log_pages,entries);
3050                         max_entries = softc->valid_iddir_len - entries_offset;
3051                         if ((softc->valid_iddir_len > (entries_offset + 1))
3052                          && (le64dec(softc->ata_iddir.header) ==
3053                              ATA_IDLOG_REVISION)
3054                          && (softc->ata_iddir.entry_count > 0)) {
3055                                 int num_entries, i;
3056
3057                                 num_entries = softc->ata_iddir.entry_count;
3058                                 num_entries = min(num_entries,
3059                                    softc->valid_iddir_len - entries_offset);
3060                                 for (i = 0; i < num_entries &&
3061                                      i < max_entries; i++) {
3062                                         if (softc->ata_iddir.entries[i] ==
3063                                             ATA_IDL_SUP_CAP)
3064                                                 softc->flags |=
3065                                                     ADA_FLAG_CAN_SUPCAP;
3066                                         else if (softc->ata_iddir.entries[i]==
3067                                                  ATA_IDL_ZDI)
3068                                                 softc->flags |=
3069                                                     ADA_FLAG_CAN_ZONE;
3070
3071                                         if ((softc->flags &
3072                                              ADA_FLAG_CAN_SUPCAP)
3073                                          && (softc->flags &
3074                                              ADA_FLAG_CAN_ZONE))
3075                                                 break;
3076                                 }
3077                         }
3078                 } else {
3079                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3080                                          SF_RETRY_UA|SF_NO_PRINT);
3081                         if (error == ERESTART)
3082                                 return;
3083                         else if (error != 0) {
3084                                 /*
3085                                  * If we can't get the ATA Identify Data log
3086                                  * directory, then it effectively isn't
3087                                  * supported even if the ATA Log directory
3088                                  * a non-zero number of pages present for
3089                                  * this log.
3090                                  */
3091                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3092                                 if ((done_ccb->ccb_h.status &
3093                                      CAM_DEV_QFRZN) != 0) {
3094                                         /* Don't wedge this device's queue */
3095                                         cam_release_devq(done_ccb->ccb_h.path,
3096                                                          /*relsim_flags*/0,
3097                                                          /*reduction*/0,
3098                                                          /*timeout*/0,
3099                                                          /*getcount_only*/0);
3100                                 }
3101                         }
3102                 }
3103
3104                 free(ataio->data_ptr, M_ATADA);
3105
3106                 if ((error == 0)
3107                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3108                         softc->state = ADA_STATE_SUP_CAP;
3109                         xpt_release_ccb(done_ccb);
3110                         xpt_schedule(periph, priority);
3111                 } else
3112                         adaprobedone(periph, done_ccb);
3113                 return;
3114         }
3115         case ADA_CCB_SUP_CAP: {
3116                 int error;
3117
3118                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3119                         uint32_t valid_len;
3120                         size_t needed_size;
3121                         struct ata_identify_log_sup_cap *sup_cap;
3122                         error = 0;
3123
3124                         sup_cap = (struct ata_identify_log_sup_cap *)
3125                             ataio->data_ptr;
3126                         valid_len = ataio->dxfer_len - ataio->resid;
3127                         needed_size =
3128                             __offsetof(struct ata_identify_log_sup_cap,
3129                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3130                         if (valid_len >= needed_size) {
3131                                 uint64_t zoned, zac_cap;
3132
3133                                 zoned = le64dec(sup_cap->zoned_cap);
3134                                 if (zoned & ATA_ZONED_VALID) {
3135                                         /*
3136                                          * This should have already been
3137                                          * set, because this is also in the
3138                                          * ATA identify data.
3139                                          */
3140                                         if ((zoned & ATA_ZONED_MASK) ==
3141                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3142                                                 softc->zone_mode =
3143                                                     ADA_ZONE_HOST_AWARE;
3144                                         else if ((zoned & ATA_ZONED_MASK) ==
3145                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3146                                                 softc->zone_mode =
3147                                                     ADA_ZONE_DRIVE_MANAGED;
3148                                 }
3149
3150                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3151                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3152                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3153                                                 softc->zone_flags |=
3154                                                     ADA_ZONE_FLAG_RZ_SUP;
3155                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3156                                                 softc->zone_flags |=
3157                                                     ADA_ZONE_FLAG_OPEN_SUP;
3158                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3159                                                 softc->zone_flags |=
3160                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3161                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3162                                                 softc->zone_flags |=
3163                                                     ADA_ZONE_FLAG_FINISH_SUP;
3164                                         if (zac_cap & ATA_ND_RWP_SUP)
3165                                                 softc->zone_flags |=
3166                                                     ADA_ZONE_FLAG_RWP_SUP;
3167                                 } else {
3168                                         /*
3169                                          * This field was introduced in
3170                                          * ACS-4, r08 on April 28th, 2015.
3171                                          * If the drive firmware was written
3172                                          * to an earlier spec, it won't have
3173                                          * the field.  So, assume all
3174                                          * commands are supported.
3175                                          */
3176                                         softc->zone_flags |=
3177                                             ADA_ZONE_FLAG_SUP_MASK;
3178                                 }
3179                         }
3180                 } else {
3181                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3182                                          SF_RETRY_UA|SF_NO_PRINT);
3183                         if (error == ERESTART)
3184                                 return;
3185                         else if (error != 0) {
3186                                 /*
3187                                  * If we can't get the ATA Identify Data
3188                                  * Supported Capabilities page, clear the
3189                                  * flag...
3190                                  */
3191                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3192                                 /*
3193                                  * And clear zone capabilities.
3194                                  */
3195                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3196                                 if ((done_ccb->ccb_h.status &
3197                                      CAM_DEV_QFRZN) != 0) {
3198                                         /* Don't wedge this device's queue */
3199                                         cam_release_devq(done_ccb->ccb_h.path,
3200                                                          /*relsim_flags*/0,
3201                                                          /*reduction*/0,
3202                                                          /*timeout*/0,
3203                                                          /*getcount_only*/0);
3204                                 }
3205                         }
3206                 }
3207
3208                 free(ataio->data_ptr, M_ATADA);
3209
3210                 if ((error == 0)
3211                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3212                         softc->state = ADA_STATE_ZONE;
3213                         xpt_release_ccb(done_ccb);
3214                         xpt_schedule(periph, priority);
3215                 } else
3216                         adaprobedone(periph, done_ccb);
3217                 return;
3218         }
3219         case ADA_CCB_ZONE: {
3220                 int error;
3221
3222                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3223                         struct ata_zoned_info_log *zi_log;
3224                         uint32_t valid_len;
3225                         size_t needed_size;
3226
3227                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3228
3229                         valid_len = ataio->dxfer_len - ataio->resid;
3230                         needed_size = __offsetof(struct ata_zoned_info_log,
3231                             version_info) + 1 + sizeof(zi_log->version_info);
3232                         if (valid_len >= needed_size) {
3233                                 uint64_t tmpvar;
3234
3235                                 tmpvar = le64dec(zi_log->zoned_cap);
3236                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3237                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3238                                                 softc->zone_flags |=
3239                                                     ADA_ZONE_FLAG_URSWRZ;
3240                                         else
3241                                                 softc->zone_flags &=
3242                                                     ~ADA_ZONE_FLAG_URSWRZ;
3243                                 }
3244                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3245                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3246                                         softc->zone_flags |=
3247                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3248                                         softc->optimal_seq_zones = (tmpvar &
3249                                             ATA_ZDI_OPT_SEQ_MASK);
3250                                 } else {
3251                                         softc->zone_flags &=
3252                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3253                                         softc->optimal_seq_zones = 0;
3254                                 }
3255
3256                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3257                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3258                                         softc->zone_flags |=
3259                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3260                                         softc->optimal_nonseq_zones =
3261                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3262                                 } else {
3263                                         softc->zone_flags &=
3264                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3265                                         softc->optimal_nonseq_zones = 0;
3266                                 }
3267
3268                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3269                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3270                                         softc->zone_flags |=
3271                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3272                                         softc->max_seq_zones =
3273                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3274                                 } else {
3275                                         softc->zone_flags &=
3276                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3277                                         softc->max_seq_zones = 0;
3278                                 }
3279                         }
3280                 } else {
3281                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3282                                          SF_RETRY_UA|SF_NO_PRINT);
3283                         if (error == ERESTART)
3284                                 return;
3285                         else if (error != 0) {
3286                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3287                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3288
3289                                 if ((done_ccb->ccb_h.status &
3290                                      CAM_DEV_QFRZN) != 0) {
3291                                         /* Don't wedge this device's queue */
3292                                         cam_release_devq(done_ccb->ccb_h.path,
3293                                                          /*relsim_flags*/0,
3294                                                          /*reduction*/0,
3295                                                          /*timeout*/0,
3296                                                          /*getcount_only*/0);
3297                                 }
3298                         }
3299                 }
3300                 free(ataio->data_ptr, M_ATADA);
3301
3302                 adaprobedone(periph, done_ccb);
3303                 return;
3304         }
3305         case ADA_CCB_DUMP:
3306                 /* No-op.  We're polling */
3307                 return;
3308         default:
3309                 break;
3310         }
3311         xpt_release_ccb(done_ccb);
3312 }
3313
3314 static int
3315 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3316 {
3317 #ifdef CAM_IO_STATS
3318         struct ada_softc *softc;
3319         struct cam_periph *periph;
3320
3321         periph = xpt_path_periph(ccb->ccb_h.path);
3322         softc = (struct ada_softc *)periph->softc;
3323
3324         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3325         case CAM_CMD_TIMEOUT:
3326                 softc->timeouts++;
3327                 break;
3328         case CAM_REQ_ABORTED:
3329         case CAM_REQ_CMP_ERR:
3330         case CAM_REQ_TERMIO:
3331         case CAM_UNREC_HBA_ERROR:
3332         case CAM_DATA_RUN_ERR:
3333         case CAM_ATA_STATUS_ERROR:
3334                 softc->errors++;
3335                 break;
3336         default:
3337                 break;
3338         }
3339 #endif
3340
3341         return(cam_periph_error(ccb, cam_flags, sense_flags));
3342 }
3343
3344 static void
3345 adasetgeom(struct ada_softc *softc, struct ccb_getdev *cgd)
3346 {
3347         struct disk_params *dp = &softc->params;
3348         u_int64_t lbasize48;
3349         u_int32_t lbasize;
3350         u_int maxio, d_flags;
3351
3352         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3353         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3354             cgd->ident_data.current_heads != 0 &&
3355             cgd->ident_data.current_sectors != 0) {
3356                 dp->heads = cgd->ident_data.current_heads;
3357                 dp->secs_per_track = cgd->ident_data.current_sectors;
3358                 dp->cylinders = cgd->ident_data.cylinders;
3359                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3360                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3361         } else {
3362                 dp->heads = cgd->ident_data.heads;
3363                 dp->secs_per_track = cgd->ident_data.sectors;
3364                 dp->cylinders = cgd->ident_data.cylinders;
3365                 dp->sectors = cgd->ident_data.cylinders *
3366                               (u_int32_t)(dp->heads * dp->secs_per_track);
3367         }
3368         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3369                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3370
3371         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3372         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3373                 dp->sectors = lbasize;
3374
3375         /* use the 48bit LBA size if valid */
3376         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3377                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3378                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3379                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3380         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3381             lbasize48 > ATA_MAX_28BIT_LBA)
3382                 dp->sectors = lbasize48;
3383
3384         maxio = softc->cpi.maxio;               /* Honor max I/O size of SIM */
3385         if (maxio == 0)
3386                 maxio = DFLTPHYS;       /* traditional default */
3387         else if (maxio > MAXPHYS)
3388                 maxio = MAXPHYS;        /* for safety */
3389         if (softc->flags & ADA_FLAG_CAN_48BIT)
3390                 maxio = min(maxio, 65536 * softc->params.secsize);
3391         else                                    /* 28bit ATA command limit */
3392                 maxio = min(maxio, 256 * softc->params.secsize);
3393         if (softc->quirks & ADA_Q_128KB)
3394                 maxio = min(maxio, 128 * 1024);
3395         softc->disk->d_maxsize = maxio;
3396         d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
3397         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
3398                 d_flags |= DISKFLAG_CANFLUSHCACHE;
3399         if (softc->flags & ADA_FLAG_CAN_TRIM) {
3400                 d_flags |= DISKFLAG_CANDELETE;
3401                 softc->disk->d_delmaxsize = softc->params.secsize *
3402                     ATA_DSM_RANGE_MAX * softc->trim_max_ranges;
3403         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
3404             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
3405                 d_flags |= DISKFLAG_CANDELETE;
3406                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
3407         } else
3408                 softc->disk->d_delmaxsize = maxio;
3409         if ((softc->cpi.hba_misc & PIM_UNMAPPED) != 0) {
3410                 d_flags |= DISKFLAG_UNMAPPED_BIO;
3411                 softc->unmappedio = 1;
3412         }
3413         softc->disk->d_flags = d_flags;
3414         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
3415             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
3416         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
3417             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
3418
3419         softc->disk->d_sectorsize = softc->params.secsize;
3420         softc->disk->d_mediasize = (off_t)softc->params.sectors *
3421             softc->params.secsize;
3422         if (ata_physical_sector_size(&cgd->ident_data) !=
3423             softc->params.secsize) {
3424                 softc->disk->d_stripesize =
3425                     ata_physical_sector_size(&cgd->ident_data);
3426                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
3427                     ata_logical_sector_offset(&cgd->ident_data)) %
3428                     softc->disk->d_stripesize;
3429         } else if (softc->quirks & ADA_Q_4K) {
3430                 softc->disk->d_stripesize = 4096;
3431                 softc->disk->d_stripeoffset = 0;
3432         }
3433         softc->disk->d_fwsectors = softc->params.secs_per_track;
3434         softc->disk->d_fwheads = softc->params.heads;
3435         ata_disk_firmware_geom_adjust(softc->disk);
3436         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
3437         snprintf(softc->disk->d_attachment, sizeof(softc->disk->d_attachment),
3438             "%s%d", softc->cpi.dev_name, softc->cpi.unit_number);
3439 }
3440
3441 static void
3442 adasendorderedtag(void *arg)
3443 {
3444         struct ada_softc *softc = arg;
3445
3446         if (ada_send_ordered) {
3447                 if (softc->outstanding_cmds > 0) {
3448                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3449                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3450                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3451                 }
3452         }
3453         /* Queue us up again */
3454         callout_reset(&softc->sendordered_c,
3455             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3456             adasendorderedtag, softc);
3457 }
3458
3459 /*
3460  * Step through all ADA peripheral drivers, and if the device is still open,
3461  * sync the disk cache to physical media.
3462  */
3463 static void
3464 adaflush(void)
3465 {
3466         struct cam_periph *periph;
3467         struct ada_softc *softc;
3468         union ccb *ccb;
3469         int error;
3470
3471         CAM_PERIPH_FOREACH(periph, &adadriver) {
3472                 softc = (struct ada_softc *)periph->softc;
3473                 if (SCHEDULER_STOPPED()) {
3474                         /* If we paniced with the lock held, do not recurse. */
3475                         if (!cam_periph_owned(periph) &&
3476                             (softc->flags & ADA_FLAG_OPEN)) {
3477                                 adadump(softc->disk, NULL, 0, 0, 0);
3478                         }
3479                         continue;
3480                 }
3481                 cam_periph_lock(periph);
3482                 /*
3483                  * We only sync the cache if the drive is still open, and
3484                  * if the drive is capable of it..
3485                  */
3486                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3487                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3488                         cam_periph_unlock(periph);
3489                         continue;
3490                 }
3491
3492                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3493                 cam_fill_ataio(&ccb->ataio,
3494                                     0,
3495                                     NULL,
3496                                     CAM_DIR_NONE,
3497                                     0,
3498                                     NULL,
3499                                     0,
3500                                     ada_default_timeout*1000);
3501                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3502                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3503                 else
3504                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3505
3506                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3507                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3508                     softc->disk->d_devstat);
3509                 if (error != 0)
3510                         xpt_print(periph->path, "Synchronize cache failed\n");
3511                 xpt_release_ccb(ccb);
3512                 cam_periph_unlock(periph);
3513         }
3514 }
3515
3516 static void
3517 adaspindown(uint8_t cmd, int flags)
3518 {
3519         struct cam_periph *periph;
3520         struct ada_softc *softc;
3521         struct ccb_ataio local_ccb;
3522         int error;
3523
3524         CAM_PERIPH_FOREACH(periph, &adadriver) {
3525                 /* If we paniced with lock held - not recurse here. */
3526                 if (cam_periph_owned(periph))
3527                         continue;
3528                 cam_periph_lock(periph);
3529                 softc = (struct ada_softc *)periph->softc;
3530                 /*
3531                  * We only spin-down the drive if it is capable of it..
3532                  */
3533                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3534                         cam_periph_unlock(periph);
3535                         continue;
3536                 }
3537
3538                 if (bootverbose)
3539                         xpt_print(periph->path, "spin-down\n");
3540
3541                 memset(&local_ccb, 0, sizeof(local_ccb));
3542                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3543                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3544
3545                 cam_fill_ataio(&local_ccb,
3546                                     0,
3547                                     NULL,
3548                                     CAM_DIR_NONE | flags,
3549                                     0,
3550                                     NULL,
3551                                     0,
3552                                     ada_default_timeout*1000);
3553                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3554                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3555                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3556                     softc->disk->d_devstat);
3557                 if (error != 0)
3558                         xpt_print(periph->path, "Spin-down disk failed\n");
3559                 cam_periph_unlock(periph);
3560         }
3561 }
3562
3563 static void
3564 adashutdown(void *arg, int howto)
3565 {
3566         int how;
3567
3568         adaflush();
3569
3570         /*
3571          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3572          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3573          * a spindown. We send the former when we expect to lose power soon. For
3574          * a warm boot, we send the latter to avoid a thundering herd of spinups
3575          * just after the kernel loads while probing. We have to do something to
3576          * flush the data because the BIOS in many systems resets the HBA
3577          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3578          * as license to toss the volatile data, and others count as unclean
3579          * shutdown when in the Active PM state in SMART attributes.
3580          *
3581          * adaspindown will ensure that we don't send this to a drive that
3582          * doesn't support it.
3583          */
3584         if (ada_spindown_shutdown != 0) {
3585                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3586                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3587                 adaspindown(how, 0);
3588         }
3589 }
3590
3591 static void
3592 adasuspend(void *arg)
3593 {
3594
3595         adaflush();
3596         /*
3597          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3598          * so we don't need to send it as well.
3599          */
3600         if (ada_spindown_suspend != 0)
3601                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3602 }
3603
3604 static void
3605 adaresume(void *arg)
3606 {
3607         struct cam_periph *periph;
3608         struct ada_softc *softc;
3609
3610         if (ada_spindown_suspend == 0)
3611                 return;
3612
3613         CAM_PERIPH_FOREACH(periph, &adadriver) {
3614                 cam_periph_lock(periph);
3615                 softc = (struct ada_softc *)periph->softc;
3616                 /*
3617                  * We only spin-down the drive if it is capable of it..
3618                  */
3619                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3620                         cam_periph_unlock(periph);
3621                         continue;
3622                 }
3623
3624                 if (bootverbose)
3625                         xpt_print(periph->path, "resume\n");
3626
3627                 /*
3628                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3629                  * sleep request.
3630                  */
3631                 cam_release_devq(periph->path,
3632                          /*relsim_flags*/0,
3633                          /*openings*/0,
3634                          /*timeout*/0,
3635                          /*getcount_only*/0);
3636
3637                 cam_periph_unlock(periph);
3638         }
3639 }
3640
3641 #endif /* _KERNEL */