sys/cam/ata/ata_da.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer,
  12  *    without modification, immediately at the beginning of the file.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include "opt_ada.h"
  33
  34 #include <sys/param.h>
  35
  36 #ifdef _KERNEL
  37 #include <sys/systm.h>
  38 #include <sys/kernel.h>
  39 #include <sys/bio.h>
  40 #include <sys/sysctl.h>
  41 #include <sys/taskqueue.h>
  42 #include <sys/lock.h>
  43 #include <sys/mutex.h>
  44 #include <sys/conf.h>
  45 #include <sys/devicestat.h>
  46 #include <sys/eventhandler.h>
  47 #include <sys/malloc.h>
  48 #include <sys/endian.h>
  49 #include <sys/cons.h>
  50 #include <sys/proc.h>
  51 #include <sys/reboot.h>
  52 #include <sys/sbuf.h>
  53 #include <geom/geom_disk.h>
  54 #endif /* _KERNEL */
  55
  56 #ifndef _KERNEL
  57 #include <stdio.h>
  58 #include <string.h>
  59 #endif /* _KERNEL */
  60
  61 #include <cam/cam.h>
  62 #include <cam/cam_ccb.h>
  63 #include <cam/cam_periph.h>
  64 #include <cam/cam_xpt_periph.h>
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_da.h>
  67 #include <cam/cam_sim.h>
  68 #include <cam/cam_iosched.h>
  69
  70 #include <cam/ata/ata_all.h>
  71
  72 #include <machine/md_var.h>     /* geometry translation */
  73
  74 #ifdef _KERNEL
  75
  76 #define ATA_MAX_28BIT_LBA               268435455UL
  77
  78 extern int iosched_debug;
  79
  80 typedef enum {
  81         ADA_STATE_RAHEAD,
  82         ADA_STATE_WCACHE,
  83         ADA_STATE_LOGDIR,
  84         ADA_STATE_IDDIR,
  85         ADA_STATE_SUP_CAP,
  86         ADA_STATE_ZONE,
  87         ADA_STATE_NORMAL
  88 } ada_state;
  89
  90 typedef enum {
  91         ADA_FLAG_CAN_48BIT      = 0x00000002,
  92         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  93         ADA_FLAG_CAN_NCQ        = 0x00000008,
  94         ADA_FLAG_CAN_DMA        = 0x00000010,
  95         ADA_FLAG_NEED_OTAG      = 0x00000020,
  96         ADA_FLAG_WAS_OTAG       = 0x00000040,
  97         ADA_FLAG_CAN_TRIM       = 0x00000080,
  98         ADA_FLAG_OPEN           = 0x00000100,
  99         ADA_FLAG_SCTX_INIT      = 0x00000200,
 100         ADA_FLAG_CAN_CFA        = 0x00000400,
 101         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 102         ADA_FLAG_CAN_DMA48      = 0x00001000,
 103         ADA_FLAG_CAN_LOG        = 0x00002000,
 104         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 105         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 106         ADA_FLAG_CAN_ZONE       = 0x00010000,
 107         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 108         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 109         ADA_FLAG_PROBED         = 0x00080000,
 110         ADA_FLAG_ANNOUNCED      = 0x00100000,
 111         ADA_FLAG_DIRTY          = 0x00200000,
 112         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 113         ADA_FLAG_PIM_ATA_EXT    = 0x00800000
 114 } ada_flags;
 115
 116 typedef enum {
 117         ADA_Q_NONE              = 0x00,
 118         ADA_Q_4K                = 0x01,
 119         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 120         ADA_Q_LOG_BROKEN        = 0x04,
 121         ADA_Q_SMR_DM            = 0x08
 122 } ada_quirks;
 123
 124 #define ADA_Q_BIT_STRING        \
 125         "\020"                  \
 126         "\0014K"                \
 127         "\002NCQ_TRIM_BROKEN"   \
 128         "\003LOG_BROKEN"        \
 129         "\004SMR_DM"
 130
 131 typedef enum {
 132         ADA_CCB_RAHEAD          = 0x01,
 133         ADA_CCB_WCACHE          = 0x02,
 134         ADA_CCB_BUFFER_IO       = 0x03,
 135         ADA_CCB_DUMP            = 0x05,
 136         ADA_CCB_TRIM            = 0x06,
 137         ADA_CCB_LOGDIR          = 0x07,
 138         ADA_CCB_IDDIR           = 0x08,
 139         ADA_CCB_SUP_CAP         = 0x09,
 140         ADA_CCB_ZONE            = 0x0a,
 141         ADA_CCB_TYPE_MASK       = 0x0F,
 142 } ada_ccb_state;
 143
 144 typedef enum {
 145         ADA_ZONE_NONE           = 0x00,
 146         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 147         ADA_ZONE_HOST_AWARE     = 0x02,
 148         ADA_ZONE_HOST_MANAGED   = 0x03
 149 } ada_zone_mode;
 150
 151 typedef enum {
 152         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 153         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 154         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 155         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 156         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 157         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 158                                            ADA_ZONE_FLAG_OPEN_SUP |
 159                                            ADA_ZONE_FLAG_CLOSE_SUP |
 160                                            ADA_ZONE_FLAG_FINISH_SUP |
 161                                            ADA_ZONE_FLAG_RWP_SUP),
 162         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 163         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 164         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 165         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 166         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 167                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 168                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 169 } ada_zone_flags;
 170
 171 static struct ada_zone_desc {
 172         ada_zone_flags value;
 173         const char *desc;
 174 } ada_zone_desc_table[] = {
 175         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 176         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 177         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 178         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 179         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 180 };
 181
 182
 183 /* Offsets into our private area for storing information */
 184 #define ccb_state       ppriv_field0
 185 #define ccb_bp          ppriv_ptr1
 186
 187 typedef enum {
 188         ADA_DELETE_NONE,
 189         ADA_DELETE_DISABLE,
 190         ADA_DELETE_CFA_ERASE,
 191         ADA_DELETE_DSM_TRIM,
 192         ADA_DELETE_NCQ_DSM_TRIM,
 193         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 194         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 195 } ada_delete_methods;
 196
 197 static const char *ada_delete_method_names[] =
 198     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 199 #if 0
 200 static const char *ada_delete_method_desc[] =
 201     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 202 #endif
 203
 204 struct disk_params {
 205         u_int8_t  heads;
 206         u_int8_t  secs_per_track;
 207         u_int32_t cylinders;
 208         u_int32_t secsize;      /* Number of bytes/logical sector */
 209         u_int64_t sectors;      /* Total number sectors */
 210 };
 211
 212 #define TRIM_MAX_BLOCKS 8
 213 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 214 struct trim_request {
 215         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 216         TAILQ_HEAD(, bio) bps;
 217 };
 218
 219 struct ada_softc {
 220         struct   cam_iosched_softc *cam_iosched;
 221         int      outstanding_cmds;      /* Number of active commands */
 222         int      refcount;              /* Active xpt_action() calls */
 223         ada_state state;
 224         ada_flags flags;
 225         ada_zone_mode zone_mode;
 226         ada_zone_flags zone_flags;
 227         struct ata_gp_log_dir ata_logdir;
 228         int valid_logdir_len;
 229         struct ata_identify_log_pages ata_iddir;
 230         int valid_iddir_len;
 231         uint64_t optimal_seq_zones;
 232         uint64_t optimal_nonseq_zones;
 233         uint64_t max_seq_zones;
 234         ada_quirks quirks;
 235         ada_delete_methods delete_method;
 236         int      trim_max_ranges;
 237         int      read_ahead;
 238         int      write_cache;
 239         int      unmappedio;
 240         int      rotating;
 241 #ifdef CAM_TEST_FAILURE
 242         int      force_read_error;
 243         int      force_write_error;
 244         int      periodic_read_error;
 245         int      periodic_read_count;
 246 #endif
 247         struct   disk_params params;
 248         struct   disk *disk;
 249         struct task             sysctl_task;
 250         struct sysctl_ctx_list  sysctl_ctx;
 251         struct sysctl_oid       *sysctl_tree;
 252         struct callout          sendordered_c;
 253         struct trim_request     trim_req;
 254 #ifdef CAM_IO_STATS
 255         struct sysctl_ctx_list  sysctl_stats_ctx;
 256         struct sysctl_oid       *sysctl_stats_tree;
 257         u_int   timeouts;
 258         u_int   errors;
 259         u_int   invalidations;
 260 #endif
 261 #define ADA_ANNOUNCETMP_SZ 80
 262         char    announce_temp[ADA_ANNOUNCETMP_SZ];
 263 #define ADA_ANNOUNCE_SZ 400
 264         char    announce_buffer[ADA_ANNOUNCE_SZ];
 265 };
 266
 267 struct ada_quirk_entry {
 268         struct scsi_inquiry_pattern inq_pat;
 269         ada_quirks quirks;
 270 };
 271
 272 static struct ada_quirk_entry ada_quirk_table[] =
 273 {
 274         {
 275                 /* Hitachi Advanced Format (4k) drives */
 276                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 277                 /*quirks*/ADA_Q_4K
 278         },
 279         {
 280                 /* Samsung Advanced Format (4k) drives */
 281                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 282                 /*quirks*/ADA_Q_4K
 283         },
 284         {
 285                 /* Samsung Advanced Format (4k) drives */
 286                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 287                 /*quirks*/ADA_Q_4K
 288         },
 289         {
 290                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 291                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 292                 /*quirks*/ADA_Q_4K
 293         },
 294         {
 295                 /* Seagate Barracuda Advanced Format (4k) drives */
 296                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 297                 /*quirks*/ADA_Q_4K
 298         },
 299         {
 300                 /* Seagate Barracuda Advanced Format (4k) drives */
 301                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 302                 /*quirks*/ADA_Q_4K
 303         },
 304         {
 305                 /* Seagate Momentus Advanced Format (4k) drives */
 306                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 307                 /*quirks*/ADA_Q_4K
 308         },
 309         {
 310                 /* Seagate Momentus Advanced Format (4k) drives */
 311                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 312                 /*quirks*/ADA_Q_4K
 313         },
 314         {
 315                 /* Seagate Momentus Advanced Format (4k) drives */
 316                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 317                 /*quirks*/ADA_Q_4K
 318         },
 319         {
 320                 /* Seagate Momentus Advanced Format (4k) drives */
 321                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 322                 /*quirks*/ADA_Q_4K
 323         },
 324         {
 325                 /* Seagate Momentus Advanced Format (4k) drives */
 326                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 327                 /*quirks*/ADA_Q_4K
 328         },
 329         {
 330                 /* Seagate Momentus Advanced Format (4k) drives */
 331                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 332                 /*quirks*/ADA_Q_4K
 333         },
 334         {
 335                 /* Seagate Momentus Advanced Format (4k) drives */
 336                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 337                 /*quirks*/ADA_Q_4K
 338         },
 339         {
 340                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 341                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 342                 /*quirks*/ADA_Q_4K
 343         },
 344         {
 345                 /* WDC Caviar Red Advanced Format (4k) drives */
 346                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 347                 /*quirks*/ADA_Q_4K
 348         },
 349         {
 350                 /* WDC Caviar Green Advanced Format (4k) drives */
 351                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 352                 /*quirks*/ADA_Q_4K
 353         },
 354         {
 355                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 356                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 357                 /*quirks*/ADA_Q_4K
 358         },
 359         {
 360                 /* WDC Caviar Red Advanced Format (4k) drives */
 361                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 362                 /*quirks*/ADA_Q_4K
 363         },
 364         {
 365                 /* WDC Caviar Black Advanced Format (4k) drives */
 366                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????AZEX*", "*" },
 367                 /*quirks*/ADA_Q_4K
 368         },
 369         {
 370                 /* WDC Caviar Black Advanced Format (4k) drives */
 371                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????FZEX*", "*" },
 372                 /*quirks*/ADA_Q_4K
 373         },
 374         {
 375                 /* WDC Caviar Green Advanced Format (4k) drives */
 376                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 377                 /*quirks*/ADA_Q_4K
 378         },
 379         {
 380                 /* WDC Caviar Green Advanced Format (4k) drives */
 381                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 382                 /*quirks*/ADA_Q_4K
 383         },
 384         {
 385                 /* WDC Scorpio Black Advanced Format (4k) drives */
 386                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 387                 /*quirks*/ADA_Q_4K
 388         },
 389         {
 390                 /* WDC Scorpio Black Advanced Format (4k) drives */
 391                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 392                 /*quirks*/ADA_Q_4K
 393         },
 394         {
 395                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 396                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 397                 /*quirks*/ADA_Q_4K
 398         },
 399         {
 400                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 401                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 402                 /*quirks*/ADA_Q_4K
 403         },
 404         /* SSDs */
 405         {
 406                 /*
 407                  * Corsair Force 2 SSDs
 408                  * 4k optimised & trim only works in 4k requests + 4k aligned
 409                  */
 410                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 411                 /*quirks*/ADA_Q_4K
 412         },
 413         {
 414                 /*
 415                  * Corsair Force 3 SSDs
 416                  * 4k optimised & trim only works in 4k requests + 4k aligned
 417                  */
 418                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 419                 /*quirks*/ADA_Q_4K
 420         },
 421         {
 422                 /*
 423                  * Corsair Neutron GTX SSDs
 424                  * 4k optimised & trim only works in 4k requests + 4k aligned
 425                  */
 426                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 427                 /*quirks*/ADA_Q_4K
 428         },
 429         {
 430                 /*
 431                  * Corsair Force GT & GS SSDs
 432                  * 4k optimised & trim only works in 4k requests + 4k aligned
 433                  */
 434                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 435                 /*quirks*/ADA_Q_4K
 436         },
 437         {
 438                 /*
 439                  * Crucial M4 SSDs
 440                  * 4k optimised & trim only works in 4k requests + 4k aligned
 441                  */
 442                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 443                 /*quirks*/ADA_Q_4K
 444         },
 445         {
 446                 /*
 447                  * Crucial M500 SSDs MU07 firmware
 448                  * NCQ Trim works
 449                  */
 450                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 451                 /*quirks*/0
 452         },
 453         {
 454                 /*
 455                  * Crucial M500 SSDs all other firmware
 456                  * NCQ Trim doesn't work
 457                  */
 458                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 459                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 460         },
 461         {
 462                 /*
 463                  * Crucial M550 SSDs
 464                  * NCQ Trim doesn't work, but only on MU01 firmware
 465                  */
 466                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 467                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 468         },
 469         {
 470                 /*
 471                  * Crucial MX100 SSDs
 472                  * NCQ Trim doesn't work, but only on MU01 firmware
 473                  */
 474                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 475                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 476         },
 477         {
 478                 /*
 479                  * Crucial RealSSD C300 SSDs
 480                  * 4k optimised
 481                  */
 482                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 483                 "*" }, /*quirks*/ADA_Q_4K
 484         },
 485         {
 486                 /*
 487                  * FCCT M500 SSDs
 488                  * NCQ Trim doesn't work
 489                  */
 490                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 491                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 492         },
 493         {
 494                 /*
 495                  * Intel 320 Series SSDs
 496                  * 4k optimised & trim only works in 4k requests + 4k aligned
 497                  */
 498                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 499                 /*quirks*/ADA_Q_4K
 500         },
 501         {
 502                 /*
 503                  * Intel 330 Series SSDs
 504                  * 4k optimised & trim only works in 4k requests + 4k aligned
 505                  */
 506                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 507                 /*quirks*/ADA_Q_4K
 508         },
 509         {
 510                 /*
 511                  * Intel 510 Series SSDs
 512                  * 4k optimised & trim only works in 4k requests + 4k aligned
 513                  */
 514                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 515                 /*quirks*/ADA_Q_4K
 516         },
 517         {
 518                 /*
 519                  * Intel 520 Series SSDs
 520                  * 4k optimised & trim only works in 4k requests + 4k aligned
 521                  */
 522                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 523                 /*quirks*/ADA_Q_4K
 524         },
 525         {
 526                 /*
 527                  * Intel S3610 Series SSDs
 528                  * 4k optimised & trim only works in 4k requests + 4k aligned
 529                  */
 530                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BX*", "*" },
 531                 /*quirks*/ADA_Q_4K
 532         },
 533         {
 534                 /*
 535                  * Intel X25-M Series SSDs
 536                  * 4k optimised & trim only works in 4k requests + 4k aligned
 537                  */
 538                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 539                 /*quirks*/ADA_Q_4K
 540         },
 541         {
 542                 /*
 543                  * Kingston E100 Series SSDs
 544                  * 4k optimised & trim only works in 4k requests + 4k aligned
 545                  */
 546                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 547                 /*quirks*/ADA_Q_4K
 548         },
 549         {
 550                 /*
 551                  * Kingston HyperX 3k SSDs
 552                  * 4k optimised & trim only works in 4k requests + 4k aligned
 553                  */
 554                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 555                 /*quirks*/ADA_Q_4K
 556         },
 557         {
 558                 /*
 559                  * Marvell SSDs (entry taken from OpenSolaris)
 560                  * 4k optimised & trim only works in 4k requests + 4k aligned
 561                  */
 562                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 563                 /*quirks*/ADA_Q_4K
 564         },
 565         {
 566                 /*
 567                  * Micron M500 SSDs firmware MU07
 568                  * NCQ Trim works?
 569                  */
 570                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 571                 /*quirks*/0
 572         },
 573         {
 574                 /*
 575                  * Micron M500 SSDs all other firmware
 576                  * NCQ Trim doesn't work
 577                  */
 578                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 579                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 580         },
 581         {
 582                 /*
 583                  * Micron M5[15]0 SSDs
 584                  * NCQ Trim doesn't work, but only MU01 firmware
 585                  */
 586                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 587                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 588         },
 589         {
 590                 /*
 591                  * Micron 5100 SSDs
 592                  * 4k optimised & trim only works in 4k requests + 4k aligned
 593                  */
 594                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron 5100 MTFDDAK*", "*" },
 595                 /*quirks*/ADA_Q_4K
 596         },
 597         {
 598                 /*
 599                  * OCZ Agility 2 SSDs
 600                  * 4k optimised & trim only works in 4k requests + 4k aligned
 601                  */
 602                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 603                 /*quirks*/ADA_Q_4K
 604         },
 605         {
 606                 /*
 607                  * OCZ Agility 3 SSDs
 608                  * 4k optimised & trim only works in 4k requests + 4k aligned
 609                  */
 610                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 611                 /*quirks*/ADA_Q_4K
 612         },
 613         {
 614                 /*
 615                  * OCZ Deneva R Series SSDs
 616                  * 4k optimised & trim only works in 4k requests + 4k aligned
 617                  */
 618                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 619                 /*quirks*/ADA_Q_4K
 620         },
 621         {
 622                 /*
 623                  * OCZ Vertex 2 SSDs (inc pro series)
 624                  * 4k optimised & trim only works in 4k requests + 4k aligned
 625                  */
 626                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 627                 /*quirks*/ADA_Q_4K
 628         },
 629         {
 630                 /*
 631                  * OCZ Vertex 3 SSDs
 632                  * 4k optimised & trim only works in 4k requests + 4k aligned
 633                  */
 634                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 635                 /*quirks*/ADA_Q_4K
 636         },
 637         {
 638                 /*
 639                  * OCZ Vertex 4 SSDs
 640                  * 4k optimised & trim only works in 4k requests + 4k aligned
 641                  */
 642                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 643                 /*quirks*/ADA_Q_4K
 644         },
 645         {
 646                 /*
 647                  * Samsung 750 SSDs
 648                  * 4k optimised, NCQ TRIM seems to work
 649                  */
 650                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 750*", "*" },
 651                 /*quirks*/ADA_Q_4K
 652         },
 653         {
 654                 /*
 655                  * Samsung 830 Series SSDs
 656                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 657                  */
 658                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 659                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 660         },
 661         {
 662                 /*
 663                  * Samsung 840 SSDs
 664                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 665                  */
 666                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 667                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 668         },
 669         {
 670                 /*
 671                  * Samsung 845 SSDs
 672                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 673                  */
 674                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 845*", "*" },
 675                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 676         },
 677         {
 678                 /*
 679                  * Samsung 850 SSDs
 680                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 681                  */
 682                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 683                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 684         },
 685         {
 686                 /*
 687                  * Samsung SM863 Series SSDs (MZ7KM*)
 688                  * 4k optimised, NCQ believed to be working
 689                  */
 690                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 691                 /*quirks*/ADA_Q_4K
 692         },
 693         {
 694                 /*
 695                  * Samsung 843T Series SSDs (MZ7WD*)
 696                  * Samsung PM851 Series SSDs (MZ7TE*)
 697                  * Samsung PM853T Series SSDs (MZ7GE*)
 698                  * 4k optimised, NCQ believed to be broken since these are
 699                  * appear to be built with the same controllers as the 840/850.
 700                  */
 701                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 702                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 703         },
 704         {
 705                 /*
 706                  * Same as for SAMSUNG MZ7* but enable the quirks for SSD
 707                  * starting with MZ7* too
 708                  */
 709                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MZ7*", "*" },
 710                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 711         },
 712         {
 713                 /*
 714                  * Samsung PM851 Series SSDs Dell OEM
 715                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 716                  * 4k optimised, NCQ broken
 717                  */
 718                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 719                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 720         },
 721         {
 722                 /*
 723                  * SuperTalent TeraDrive CT SSDs
 724                  * 4k optimised & trim only works in 4k requests + 4k aligned
 725                  */
 726                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 727                 /*quirks*/ADA_Q_4K
 728         },
 729         {
 730                 /*
 731                  * XceedIOPS SATA SSDs
 732                  * 4k optimised
 733                  */
 734                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 735                 /*quirks*/ADA_Q_4K
 736         },
 737         {
 738                 /*
 739                  * Samsung drive that doesn't support READ LOG EXT or
 740                  * READ LOG DMA EXT, despite reporting that it does in
 741                  * ATA identify data:
 742                  * SAMSUNG HD200HJ KF100-06
 743                  */
 744                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 745                 /*quirks*/ADA_Q_LOG_BROKEN
 746         },
 747         {
 748                 /*
 749                  * Samsung drive that doesn't support READ LOG EXT or
 750                  * READ LOG DMA EXT, despite reporting that it does in
 751                  * ATA identify data:
 752                  * SAMSUNG HD501LJ CR100-10
 753                  */
 754                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 755                 /*quirks*/ADA_Q_LOG_BROKEN
 756         },
 757         {
 758                 /*
 759                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 760                  * Drive Managed SATA hard drive.  This drive doesn't report
 761                  * in firmware that it is a drive managed SMR drive.
 762                  */
 763                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS000[23]*", "*" },
 764                 /*quirks*/ADA_Q_SMR_DM
 765         },
 766         {
 767                 /* Default */
 768                 {
 769                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 770                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 771                 },
 772                 /*quirks*/0
 773         },
 774 };
 775
 776 static  disk_strategy_t adastrategy;
 777 static  dumper_t        adadump;
 778 static  periph_init_t   adainit;
 779 static  void            adadiskgonecb(struct disk *dp);
 780 static  periph_oninv_t  adaoninvalidate;
 781 static  periph_dtor_t   adacleanup;
 782 static  void            adaasync(void *callback_arg, u_int32_t code,
 783                                 struct cam_path *path, void *arg);
 784 static  int             adazonemodesysctl(SYSCTL_HANDLER_ARGS);
 785 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 786 static  void            adasysctlinit(void *context, int pending);
 787 static  int             adagetattr(struct bio *bp);
 788 static  void            adasetflags(struct ada_softc *softc,
 789                                     struct ccb_getdev *cgd);
 790 static  periph_ctor_t   adaregister;
 791 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 792                                     struct ccb_ataio *ataio);
 793 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 794                                      struct ccb_ataio *ataio);
 795 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 796 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 797                                      struct bio *bp, int *queue_ccb);
 798 static  periph_start_t  adastart;
 799 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 800 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 801 static  void            adadone(struct cam_periph *periph,
 802                                union ccb *done_ccb);
 803 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 804                                 u_int32_t sense_flags);
 805 static void             adagetparams(struct cam_periph *periph,
 806                                 struct ccb_getdev *cgd);
 807 static timeout_t        adasendorderedtag;
 808 static void             adashutdown(void *arg, int howto);
 809 static void             adasuspend(void *arg);
 810 static void             adaresume(void *arg);
 811
 812 #ifndef ADA_DEFAULT_TIMEOUT
 813 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 814 #endif
 815
 816 #ifndef ADA_DEFAULT_RETRY
 817 #define ADA_DEFAULT_RETRY       4
 818 #endif
 819
 820 #ifndef ADA_DEFAULT_SEND_ORDERED
 821 #define ADA_DEFAULT_SEND_ORDERED        1
 822 #endif
 823
 824 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 825 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 826 #endif
 827
 828 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 829 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 830 #endif
 831
 832 #ifndef ADA_DEFAULT_READ_AHEAD
 833 #define ADA_DEFAULT_READ_AHEAD  1
 834 #endif
 835
 836 #ifndef ADA_DEFAULT_WRITE_CACHE
 837 #define ADA_DEFAULT_WRITE_CACHE 1
 838 #endif
 839
 840 #define ADA_RA  (softc->read_ahead >= 0 ? \
 841                  softc->read_ahead : ada_read_ahead)
 842 #define ADA_WC  (softc->write_cache >= 0 ? \
 843                  softc->write_cache : ada_write_cache)
 844
 845 /*
 846  * Most platforms map firmware geometry to actual, but some don't.  If
 847  * not overridden, default to nothing.
 848  */
 849 #ifndef ata_disk_firmware_geom_adjust
 850 #define ata_disk_firmware_geom_adjust(disk)
 851 #endif
 852
 853 static int ada_retry_count = ADA_DEFAULT_RETRY;
 854 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 855 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 856 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 857 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 858 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 859 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 860
 861 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 862             "CAM Direct Access Disk driver");
 863 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 864            &ada_retry_count, 0, "Normal I/O retry count");
 865 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 866            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 867 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 868            &ada_send_ordered, 0, "Send Ordered Tags");
 869 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 870            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 871 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 872            &ada_spindown_suspend, 0, "Spin down upon suspend");
 873 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 874            &ada_read_ahead, 0, "Enable disk read-ahead");
 875 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 876            &ada_write_cache, 0, "Enable disk write cache");
 877
 878 /*
 879  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 880  * to the default timeout, we check to see whether an ordered
 881  * tagged transaction is appropriate to prevent simple tag
 882  * starvation.  Since we'd like to ensure that there is at least
 883  * 1/2 of the timeout length left for a starved transaction to
 884  * complete after we've sent an ordered tag, we must poll at least
 885  * four times in every timeout period.  This takes care of the worst
 886  * case where a starved transaction starts during an interval that
 887  * meets the requirement "don't send an ordered tag" test so it takes
 888  * us two intervals to determine that a tag must be sent.
 889  */
 890 #ifndef ADA_ORDEREDTAG_INTERVAL
 891 #define ADA_ORDEREDTAG_INTERVAL 4
 892 #endif
 893
 894 static struct periph_driver adadriver =
 895 {
 896         adainit, "ada",
 897         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 898 };
 899
 900 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 901
 902 PERIPHDRIVER_DECLARE(ada, adadriver);
 903
 904 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 905
 906 static int
 907 adaopen(struct disk *dp)
 908 {
 909         struct cam_periph *periph;
 910         struct ada_softc *softc;
 911         int error;
 912
 913         periph = (struct cam_periph *)dp->d_drv1;
 914         if (cam_periph_acquire(periph) != 0) {
 915                 return(ENXIO);
 916         }
 917
 918         cam_periph_lock(periph);
 919         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 920                 cam_periph_unlock(periph);
 921                 cam_periph_release(periph);
 922                 return (error);
 923         }
 924
 925         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 926             ("adaopen\n"));
 927
 928         softc = (struct ada_softc *)periph->softc;
 929         softc->flags |= ADA_FLAG_OPEN;
 930
 931         cam_periph_unhold(periph);
 932         cam_periph_unlock(periph);
 933         return (0);
 934 }
 935
 936 static int
 937 adaclose(struct disk *dp)
 938 {
 939         struct  cam_periph *periph;
 940         struct  ada_softc *softc;
 941         union ccb *ccb;
 942         int error;
 943
 944         periph = (struct cam_periph *)dp->d_drv1;
 945         softc = (struct ada_softc *)periph->softc;
 946         cam_periph_lock(periph);
 947
 948         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 949             ("adaclose\n"));
 950
 951         /* We only sync the cache if the drive is capable of it. */
 952         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 953             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 954             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 955             cam_periph_hold(periph, PRIBIO) == 0) {
 956
 957                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 958                 cam_fill_ataio(&ccb->ataio,
 959                                     1,
 960                                     NULL,
 961                                     CAM_DIR_NONE,
 962                                     0,
 963                                     NULL,
 964                                     0,
 965                                     ada_default_timeout*1000);
 966
 967                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 968                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 969                 else
 970                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 971                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 972                     /*sense_flags*/0, softc->disk->d_devstat);
 973
 974                 if (error != 0)
 975                         xpt_print(periph->path, "Synchronize cache failed\n");
 976                 softc->flags &= ~ADA_FLAG_DIRTY;
 977                 xpt_release_ccb(ccb);
 978                 cam_periph_unhold(periph);
 979         }
 980
 981         softc->flags &= ~ADA_FLAG_OPEN;
 982
 983         while (softc->refcount != 0)
 984                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
 985         cam_periph_unlock(periph);
 986         cam_periph_release(periph);
 987         return (0);
 988 }
 989
 990 static void
 991 adaschedule(struct cam_periph *periph)
 992 {
 993         struct ada_softc *softc = (struct ada_softc *)periph->softc;
 994
 995         if (softc->state != ADA_STATE_NORMAL)
 996                 return;
 997
 998         cam_iosched_schedule(softc->cam_iosched, periph);
 999 }
1000
1001 /*
1002  * Actually translate the requested transfer into one the physical driver
1003  * can understand.  The transfer is described by a buf and will include
1004  * only one physical transfer.
1005  */
1006 static void
1007 adastrategy(struct bio *bp)
1008 {
1009         struct cam_periph *periph;
1010         struct ada_softc *softc;
1011
1012         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1013         softc = (struct ada_softc *)periph->softc;
1014
1015         cam_periph_lock(periph);
1016
1017         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
1018
1019         /*
1020          * If the device has been made invalid, error out
1021          */
1022         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1023                 cam_periph_unlock(periph);
1024                 biofinish(bp, NULL, ENXIO);
1025                 return;
1026         }
1027
1028         /*
1029          * Zone commands must be ordered, because they can depend on the
1030          * effects of previously issued commands, and they may affect
1031          * commands after them.
1032          */
1033         if (bp->bio_cmd == BIO_ZONE)
1034                 bp->bio_flags |= BIO_ORDERED;
1035
1036         /*
1037          * Place it in the queue of disk activities for this disk
1038          */
1039         cam_iosched_queue_work(softc->cam_iosched, bp);
1040
1041         /*
1042          * Schedule ourselves for performing the work.
1043          */
1044         adaschedule(periph);
1045         cam_periph_unlock(periph);
1046
1047         return;
1048 }
1049
1050 static int
1051 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1052 {
1053         struct      cam_periph *periph;
1054         struct      ada_softc *softc;
1055         u_int       secsize;
1056         struct      ccb_ataio ataio;
1057         struct      disk *dp;
1058         uint64_t    lba;
1059         uint16_t    count;
1060         int         error = 0;
1061
1062         dp = arg;
1063         periph = dp->d_drv1;
1064         softc = (struct ada_softc *)periph->softc;
1065         secsize = softc->params.secsize;
1066         lba = offset / secsize;
1067         count = length / secsize;
1068
1069         if ((periph->flags & CAM_PERIPH_INVALID) != 0)
1070                 return (ENXIO);
1071
1072         memset(&ataio, 0, sizeof(ataio));
1073         if (length > 0) {
1074                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1075                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1076                 cam_fill_ataio(&ataio,
1077                     0,
1078                     NULL,
1079                     CAM_DIR_OUT,
1080                     0,
1081                     (u_int8_t *) virtual,
1082                     length,
1083                     ada_default_timeout*1000);
1084                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1085                     (lba + count >= ATA_MAX_28BIT_LBA ||
1086                     count >= 256)) {
1087                         ata_48bit_cmd(&ataio, ATA_WRITE_DMA48,
1088                             0, lba, count);
1089                 } else {
1090                         ata_28bit_cmd(&ataio, ATA_WRITE_DMA,
1091                             0, lba, count);
1092                 }
1093                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1094                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1095                 if (error != 0)
1096                         printf("Aborting dump due to I/O error.\n");
1097
1098                 return (error);
1099         }
1100
1101         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1102                 xpt_setup_ccb(&ataio.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1103
1104                 /*
1105                  * Tell the drive to flush its internal cache. if we
1106                  * can't flush in 5s we have big problems. No need to
1107                  * wait the default 60s to detect problems.
1108                  */
1109                 ataio.ccb_h.ccb_state = ADA_CCB_DUMP;
1110                 cam_fill_ataio(&ataio,
1111                                     0,
1112                                     NULL,
1113                                     CAM_DIR_NONE,
1114                                     0,
1115                                     NULL,
1116                                     0,
1117                                     5*1000);
1118
1119                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1120                         ata_48bit_cmd(&ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1121                 else
1122                         ata_28bit_cmd(&ataio, ATA_FLUSHCACHE, 0, 0, 0);
1123                 error = cam_periph_runccb((union ccb *)&ataio, adaerror,
1124                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1125                 if (error != 0)
1126                         xpt_print(periph->path, "Synchronize cache failed\n");
1127         }
1128         return (error);
1129 }
1130
1131 static void
1132 adainit(void)
1133 {
1134         cam_status status;
1135
1136         /*
1137          * Install a global async callback.  This callback will
1138          * receive async callbacks like "new device found".
1139          */
1140         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1141
1142         if (status != CAM_REQ_CMP) {
1143                 printf("ada: Failed to attach master async callback "
1144                        "due to status 0x%x!\n", status);
1145         } else if (ada_send_ordered) {
1146
1147                 /* Register our event handlers */
1148                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1149                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1150                     printf("adainit: power event registration failed!\n");
1151                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1152                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1153                     printf("adainit: power event registration failed!\n");
1154                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1155                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1156                     printf("adainit: shutdown event registration failed!\n");
1157         }
1158 }
1159
1160 /*
1161  * Callback from GEOM, called when it has finished cleaning up its
1162  * resources.
1163  */
1164 static void
1165 adadiskgonecb(struct disk *dp)
1166 {
1167         struct cam_periph *periph;
1168
1169         periph = (struct cam_periph *)dp->d_drv1;
1170
1171         cam_periph_release(periph);
1172 }
1173
1174 static void
1175 adaoninvalidate(struct cam_periph *periph)
1176 {
1177         struct ada_softc *softc;
1178
1179         softc = (struct ada_softc *)periph->softc;
1180
1181         /*
1182          * De-register any async callbacks.
1183          */
1184         xpt_register_async(0, adaasync, periph, periph->path);
1185 #ifdef CAM_IO_STATS
1186         softc->invalidations++;
1187 #endif
1188
1189         /*
1190          * Return all queued I/O with ENXIO.
1191          * XXX Handle any transactions queued to the card
1192          *     with XPT_ABORT_CCB.
1193          */
1194         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1195
1196         disk_gone(softc->disk);
1197 }
1198
1199 static void
1200 adacleanup(struct cam_periph *periph)
1201 {
1202         struct ada_softc *softc;
1203
1204         softc = (struct ada_softc *)periph->softc;
1205
1206         cam_periph_unlock(periph);
1207
1208         cam_iosched_fini(softc->cam_iosched);
1209
1210         /*
1211          * If we can't free the sysctl tree, oh well...
1212          */
1213         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1214 #ifdef CAM_IO_STATS
1215                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1216                         xpt_print(periph->path,
1217                             "can't remove sysctl stats context\n");
1218 #endif
1219                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1220                         xpt_print(periph->path,
1221                             "can't remove sysctl context\n");
1222         }
1223
1224         disk_destroy(softc->disk);
1225         callout_drain(&softc->sendordered_c);
1226         free(softc, M_DEVBUF);
1227         cam_periph_lock(periph);
1228 }
1229
1230 static void
1231 adasetdeletemethod(struct ada_softc *softc)
1232 {
1233
1234         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1235                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1236         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1237                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1238         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1239                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1240         else
1241                 softc->delete_method = ADA_DELETE_NONE;
1242 }
1243
1244 static void
1245 adaasync(void *callback_arg, u_int32_t code,
1246         struct cam_path *path, void *arg)
1247 {
1248         struct ccb_getdev cgd;
1249         struct cam_periph *periph;
1250         struct ada_softc *softc;
1251
1252         periph = (struct cam_periph *)callback_arg;
1253         switch (code) {
1254         case AC_FOUND_DEVICE:
1255         {
1256                 struct ccb_getdev *cgd;
1257                 cam_status status;
1258
1259                 cgd = (struct ccb_getdev *)arg;
1260                 if (cgd == NULL)
1261                         break;
1262
1263                 if (cgd->protocol != PROTO_ATA)
1264                         break;
1265
1266                 /*
1267                  * Allocate a peripheral instance for
1268                  * this device and start the probe
1269                  * process.
1270                  */
1271                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1272                                           adacleanup, adastart,
1273                                           "ada", CAM_PERIPH_BIO,
1274                                           path, adaasync,
1275                                           AC_FOUND_DEVICE, cgd);
1276
1277                 if (status != CAM_REQ_CMP
1278                  && status != CAM_REQ_INPROG)
1279                         printf("adaasync: Unable to attach to new device "
1280                                 "due to status 0x%x\n", status);
1281                 break;
1282         }
1283         case AC_GETDEV_CHANGED:
1284         {
1285                 softc = (struct ada_softc *)periph->softc;
1286                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1287                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1288                 xpt_action((union ccb *)&cgd);
1289
1290                 /*
1291                  * Set/clear support flags based on the new Identify data.
1292                  */
1293                 adasetflags(softc, &cgd);
1294
1295                 cam_periph_async(periph, code, path, arg);
1296                 break;
1297         }
1298         case AC_ADVINFO_CHANGED:
1299         {
1300                 uintptr_t buftype;
1301
1302                 buftype = (uintptr_t)arg;
1303                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1304                         struct ada_softc *softc;
1305
1306                         softc = periph->softc;
1307                         disk_attr_changed(softc->disk, "GEOM::physpath",
1308                                           M_NOWAIT);
1309                 }
1310                 break;
1311         }
1312         case AC_SENT_BDR:
1313         case AC_BUS_RESET:
1314         {
1315                 softc = (struct ada_softc *)periph->softc;
1316                 cam_periph_async(periph, code, path, arg);
1317                 if (softc->state != ADA_STATE_NORMAL)
1318                         break;
1319                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1320                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1321                 xpt_action((union ccb *)&cgd);
1322                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1323                         softc->state = ADA_STATE_RAHEAD;
1324                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE)
1325                         softc->state = ADA_STATE_WCACHE;
1326                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1327                       && (softc->zone_mode != ADA_ZONE_NONE))
1328                         softc->state = ADA_STATE_LOGDIR;
1329                 else
1330                     break;
1331                 if (cam_periph_acquire(periph) != 0)
1332                         softc->state = ADA_STATE_NORMAL;
1333                 else
1334                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1335         }
1336         default:
1337                 cam_periph_async(periph, code, path, arg);
1338                 break;
1339         }
1340 }
1341
1342 static int
1343 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1344 {
1345         char tmpbuf[40];
1346         struct ada_softc *softc;
1347         int error;
1348
1349         softc = (struct ada_softc *)arg1;
1350
1351         switch (softc->zone_mode) {
1352         case ADA_ZONE_DRIVE_MANAGED:
1353                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1354                 break;
1355         case ADA_ZONE_HOST_AWARE:
1356                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1357                 break;
1358         case ADA_ZONE_HOST_MANAGED:
1359                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1360                 break;
1361         case ADA_ZONE_NONE:
1362         default:
1363                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1364                 break;
1365         }
1366
1367         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1368
1369         return (error);
1370 }
1371
1372 static int
1373 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1374 {
1375         char tmpbuf[180];
1376         struct ada_softc *softc;
1377         struct sbuf sb;
1378         int error, first;
1379         unsigned int i;
1380
1381         softc = (struct ada_softc *)arg1;
1382
1383         error = 0;
1384         first = 1;
1385         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1386
1387         for (i = 0; i < sizeof(ada_zone_desc_table) /
1388              sizeof(ada_zone_desc_table[0]); i++) {
1389                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1390                         if (first == 0)
1391                                 sbuf_printf(&sb, ", ");
1392                         else
1393                                 first = 0;
1394                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1395                 }
1396         }
1397
1398         if (first == 1)
1399                 sbuf_printf(&sb, "None");
1400
1401         sbuf_finish(&sb);
1402
1403         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1404
1405         return (error);
1406 }
1407
1408
1409 static void
1410 adasysctlinit(void *context, int pending)
1411 {
1412         struct cam_periph *periph;
1413         struct ada_softc *softc;
1414         char tmpstr[32], tmpstr2[16];
1415
1416         periph = (struct cam_periph *)context;
1417
1418         /* periph was held for us when this task was enqueued */
1419         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1420                 cam_periph_release(periph);
1421                 return;
1422         }
1423
1424         softc = (struct ada_softc *)periph->softc;
1425         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1426         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1427
1428         sysctl_ctx_init(&softc->sysctl_ctx);
1429         softc->flags |= ADA_FLAG_SCTX_INIT;
1430         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
1431                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1432                 CTLFLAG_RD, 0, tmpstr, "device_index");
1433         if (softc->sysctl_tree == NULL) {
1434                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1435                 cam_periph_release(periph);
1436                 return;
1437         }
1438
1439         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1440                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1441                 softc, 0, adadeletemethodsysctl, "A",
1442                 "BIO_DELETE execution method");
1443         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1444                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1445                 &softc->read_ahead, 0, "Enable disk read ahead.");
1446         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1447                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1448                 &softc->write_cache, 0, "Enable disk write cache.");
1449         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1450                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1451                 &softc->unmappedio, 0, "Unmapped I/O leaf");
1452         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1453                 OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1454                 &softc->rotating, 0, "Rotating media");
1455         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1456                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1457                 softc, 0, adazonemodesysctl, "A",
1458                 "Zone Mode");
1459         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1460                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1461                 softc, 0, adazonesupsysctl, "A",
1462                 "Zone Support");
1463         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1464                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1465                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1466                 "Optimal Number of Open Sequential Write Preferred Zones");
1467         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1468                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1469                 "optimal_nonseq_zones", CTLFLAG_RD,
1470                 &softc->optimal_nonseq_zones,
1471                 "Optimal Number of Non-Sequentially Written Sequential Write "
1472                 "Preferred Zones");
1473         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1474                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1475                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1476                 "Maximum Number of Open Sequential Write Required Zones");
1477
1478 #ifdef CAM_TEST_FAILURE
1479         /*
1480          * Add a 'door bell' sysctl which allows one to set it from userland
1481          * and cause something bad to happen.  For the moment, we only allow
1482          * whacking the next read or write.
1483          */
1484         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1485                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1486                 &softc->force_read_error, 0,
1487                 "Force a read error for the next N reads.");
1488         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1489                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1490                 &softc->force_write_error, 0,
1491                 "Force a write error for the next N writes.");
1492         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1493                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1494                 &softc->periodic_read_error, 0,
1495                 "Force a read error every N reads (don't set too low).");
1496         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1497                 OID_AUTO, "invalidate", CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE,
1498                 periph, 0, cam_periph_invalidate_sysctl, "I",
1499                 "Write 1 to invalidate the drive immediately");
1500 #endif
1501
1502 #ifdef CAM_IO_STATS
1503         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1504                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1505                 CTLFLAG_RD, 0, "Statistics");
1506         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1507                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1508                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1509                 &softc->timeouts, 0,
1510                 "Device timeouts reported by the SIM");
1511         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1512                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1513                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1514                 &softc->errors, 0,
1515                 "Transport errors reported by the SIM.");
1516         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1517                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1518                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1519                 &softc->invalidations, 0,
1520                 "Device pack invalidations.");
1521 #endif
1522
1523         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1524             softc->sysctl_tree);
1525
1526         cam_periph_release(periph);
1527 }
1528
1529 static int
1530 adagetattr(struct bio *bp)
1531 {
1532         int ret;
1533         struct cam_periph *periph;
1534
1535         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1536         cam_periph_lock(periph);
1537         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1538             periph->path);
1539         cam_periph_unlock(periph);
1540         if (ret == 0)
1541                 bp->bio_completed = bp->bio_length;
1542         return ret;
1543 }
1544
1545 static int
1546 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1547 {
1548         char buf[16];
1549         const char *p;
1550         struct ada_softc *softc;
1551         int i, error, value, methods;
1552
1553         softc = (struct ada_softc *)arg1;
1554
1555         value = softc->delete_method;
1556         if (value < 0 || value > ADA_DELETE_MAX)
1557                 p = "UNKNOWN";
1558         else
1559                 p = ada_delete_method_names[value];
1560         strncpy(buf, p, sizeof(buf));
1561         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1562         if (error != 0 || req->newptr == NULL)
1563                 return (error);
1564         methods = 1 << ADA_DELETE_DISABLE;
1565         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1566             !(softc->flags & ADA_FLAG_CAN_48BIT))
1567                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1568         if (softc->flags & ADA_FLAG_CAN_TRIM)
1569                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1570         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1571                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1572         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1573                 if (!(methods & (1 << i)) ||
1574                     strcmp(buf, ada_delete_method_names[i]) != 0)
1575                         continue;
1576                 softc->delete_method = i;
1577                 return (0);
1578         }
1579         return (EINVAL);
1580 }
1581
1582 static void
1583 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1584 {
1585         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1586             (cgd->inq_flags & SID_DMA))
1587                 softc->flags |= ADA_FLAG_CAN_DMA;
1588         else
1589                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1590
1591         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1592                 softc->flags |= ADA_FLAG_CAN_48BIT;
1593                 if (cgd->inq_flags & SID_DMA48)
1594                         softc->flags |= ADA_FLAG_CAN_DMA48;
1595                 else
1596                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1597         } else
1598                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1599
1600         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1601                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1602         else
1603                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1604
1605         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1606                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1607         else
1608                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1609
1610         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1611             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1612                 softc->flags |= ADA_FLAG_CAN_NCQ;
1613         else
1614                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1615
1616         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1617             (cgd->inq_flags & SID_DMA)) {
1618                 softc->flags |= ADA_FLAG_CAN_TRIM;
1619                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1620                 if (cgd->ident_data.max_dsm_blocks != 0) {
1621                         softc->trim_max_ranges =
1622                             min(cgd->ident_data.max_dsm_blocks *
1623                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1624                 }
1625                 /*
1626                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1627                  * to do NCQ trims, if we support trims at all. We also need
1628                  * support from the SIM to do things properly. Perhaps we
1629                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1630                  * set too...
1631                  */
1632                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1633                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1634                     (cgd->ident_data.satacapabilities2 &
1635                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1636                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1637                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1638                 else
1639                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1640         } else
1641                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1642
1643         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1644                 softc->flags |= ADA_FLAG_CAN_CFA;
1645         else
1646                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1647
1648         /*
1649          * Now that we've set the appropriate flags, setup the delete
1650          * method.
1651          */
1652         adasetdeletemethod(softc);
1653
1654         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1655          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1656                 softc->flags |= ADA_FLAG_CAN_LOG;
1657         else
1658                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1659
1660         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1661              ATA_SUPPORT_ZONE_HOST_AWARE)
1662                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1663         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1664                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1665               || (softc->quirks & ADA_Q_SMR_DM))
1666                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1667         else
1668                 softc->zone_mode = ADA_ZONE_NONE;
1669
1670         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1671                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1672         else
1673                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1674
1675         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1676                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1677         else
1678                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1679 }
1680
1681 static cam_status
1682 adaregister(struct cam_periph *periph, void *arg)
1683 {
1684         struct ada_softc *softc;
1685         struct ccb_pathinq cpi;
1686         struct ccb_getdev *cgd;
1687         struct disk_params *dp;
1688         struct sbuf sb;
1689         char   *announce_buf;
1690         caddr_t match;
1691         u_int maxio;
1692         int quirks;
1693
1694         cgd = (struct ccb_getdev *)arg;
1695         if (cgd == NULL) {
1696                 printf("adaregister: no getdev CCB, can't register device\n");
1697                 return(CAM_REQ_CMP_ERR);
1698         }
1699
1700         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1701             M_NOWAIT|M_ZERO);
1702
1703         if (softc == NULL) {
1704                 printf("adaregister: Unable to probe new device. "
1705                     "Unable to allocate softc\n");
1706                 return(CAM_REQ_CMP_ERR);
1707         }
1708
1709         announce_buf = softc->announce_temp;
1710         bzero(announce_buf, ADA_ANNOUNCETMP_SZ);
1711
1712         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1713                 printf("adaregister: Unable to probe new device. "
1714                        "Unable to allocate iosched memory\n");
1715                 free(softc, M_DEVBUF);
1716                 return(CAM_REQ_CMP_ERR);
1717         }
1718
1719         periph->softc = softc;
1720
1721         /*
1722          * See if this device has any quirks.
1723          */
1724         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1725                                (caddr_t)ada_quirk_table,
1726                                nitems(ada_quirk_table),
1727                                sizeof(*ada_quirk_table), ata_identify_match);
1728         if (match != NULL)
1729                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1730         else
1731                 softc->quirks = ADA_Q_NONE;
1732
1733         xpt_path_inq(&cpi, periph->path);
1734
1735         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1736
1737         /*
1738          * Register this media as a disk
1739          */
1740         (void)cam_periph_hold(periph, PRIBIO);
1741         cam_periph_unlock(periph);
1742         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1743             "kern.cam.ada.%d.quirks", periph->unit_number);
1744         quirks = softc->quirks;
1745         TUNABLE_INT_FETCH(announce_buf, &quirks);
1746         softc->quirks = quirks;
1747         softc->read_ahead = -1;
1748         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1749             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1750         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1751         softc->write_cache = -1;
1752         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1753             "kern.cam.ada.%d.write_cache", periph->unit_number);
1754         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1755
1756         /*
1757          * Set support flags based on the Identify data and quirks.
1758          */
1759         adasetflags(softc, cgd);
1760
1761         /* Disable queue sorting for non-rotational media by default. */
1762         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1763                 softc->rotating = 0;
1764         } else {
1765                 softc->rotating = 1;
1766         }
1767         cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1768         adagetparams(periph, cgd);
1769         softc->disk = disk_alloc();
1770         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1771         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1772                           periph->unit_number, softc->params.secsize,
1773                           DEVSTAT_ALL_SUPPORTED,
1774                           DEVSTAT_TYPE_DIRECT |
1775                           XPORT_DEVSTAT_TYPE(cpi.transport),
1776                           DEVSTAT_PRIORITY_DISK);
1777         softc->disk->d_open = adaopen;
1778         softc->disk->d_close = adaclose;
1779         softc->disk->d_strategy = adastrategy;
1780         softc->disk->d_getattr = adagetattr;
1781         softc->disk->d_dump = adadump;
1782         softc->disk->d_gone = adadiskgonecb;
1783         softc->disk->d_name = "ada";
1784         softc->disk->d_drv1 = periph;
1785         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
1786         if (maxio == 0)
1787                 maxio = DFLTPHYS;       /* traditional default */
1788         else if (maxio > MAXPHYS)
1789                 maxio = MAXPHYS;        /* for safety */
1790         if (softc->flags & ADA_FLAG_CAN_48BIT)
1791                 maxio = min(maxio, 65536 * softc->params.secsize);
1792         else                                    /* 28bit ATA command limit */
1793                 maxio = min(maxio, 256 * softc->params.secsize);
1794         softc->disk->d_maxsize = maxio;
1795         softc->disk->d_unit = periph->unit_number;
1796         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
1797         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1798                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1799         if (softc->flags & ADA_FLAG_CAN_TRIM) {
1800                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1801                 softc->disk->d_delmaxsize = softc->params.secsize *
1802                                             ATA_DSM_RANGE_MAX *
1803                                             softc->trim_max_ranges;
1804         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1805             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1806                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1807                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1808         } else
1809                 softc->disk->d_delmaxsize = maxio;
1810         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
1811                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1812                 softc->unmappedio = 1;
1813         }
1814         if (cpi.hba_misc & PIM_ATA_EXT)
1815                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1816         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1817             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1818         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1819             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1820         softc->disk->d_hba_vendor = cpi.hba_vendor;
1821         softc->disk->d_hba_device = cpi.hba_device;
1822         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1823         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1824
1825         softc->disk->d_sectorsize = softc->params.secsize;
1826         softc->disk->d_mediasize = (off_t)softc->params.sectors *
1827             softc->params.secsize;
1828         if (ata_physical_sector_size(&cgd->ident_data) !=
1829             softc->params.secsize) {
1830                 softc->disk->d_stripesize =
1831                     ata_physical_sector_size(&cgd->ident_data);
1832                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1833                     ata_logical_sector_offset(&cgd->ident_data)) %
1834                     softc->disk->d_stripesize;
1835         } else if (softc->quirks & ADA_Q_4K) {
1836                 softc->disk->d_stripesize = 4096;
1837                 softc->disk->d_stripeoffset = 0;
1838         }
1839         softc->disk->d_fwsectors = softc->params.secs_per_track;
1840         softc->disk->d_fwheads = softc->params.heads;
1841         ata_disk_firmware_geom_adjust(softc->disk);
1842
1843         /*
1844          * Acquire a reference to the periph before we register with GEOM.
1845          * We'll release this reference once GEOM calls us back (via
1846          * adadiskgonecb()) telling us that our provider has been freed.
1847          */
1848         if (cam_periph_acquire(periph) != 0) {
1849                 xpt_print(periph->path, "%s: lost periph during "
1850                           "registration!\n", __func__);
1851                 cam_periph_lock(periph);
1852                 return (CAM_REQ_CMP_ERR);
1853         }
1854         disk_create(softc->disk, DISK_VERSION);
1855         cam_periph_lock(periph);
1856
1857         dp = &softc->params;
1858         snprintf(announce_buf, ADA_ANNOUNCETMP_SZ,
1859             "%juMB (%ju %u byte sectors)",
1860             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1861             (uintmax_t)dp->sectors, dp->secsize);
1862
1863         sbuf_new(&sb, softc->announce_buffer, ADA_ANNOUNCE_SZ, SBUF_FIXEDLEN);
1864         xpt_announce_periph_sbuf(periph, &sb, announce_buf);
1865         xpt_announce_quirks_sbuf(periph, &sb, softc->quirks, ADA_Q_BIT_STRING);
1866         sbuf_finish(&sb);
1867         sbuf_putbuf(&sb);
1868
1869         /*
1870          * Create our sysctl variables, now that we know
1871          * we have successfully attached.
1872          */
1873         if (cam_periph_acquire(periph) == 0)
1874                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1875
1876         /*
1877          * Add async callbacks for bus reset and
1878          * bus device reset calls.  I don't bother
1879          * checking if this fails as, in most cases,
1880          * the system will function just fine without
1881          * them and the only alternative would be to
1882          * not attach the device on failure.
1883          */
1884         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1885             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1886             adaasync, periph, periph->path);
1887
1888         /*
1889          * Schedule a periodic event to occasionally send an
1890          * ordered tag to a device.
1891          */
1892         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1893         callout_reset(&softc->sendordered_c,
1894             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1895             adasendorderedtag, softc);
1896
1897         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1898                 softc->state = ADA_STATE_RAHEAD;
1899         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1900                 softc->state = ADA_STATE_WCACHE;
1901         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1902                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1903                 softc->state = ADA_STATE_LOGDIR;
1904         } else {
1905                 /*
1906                  * Nothing to probe, so we can just transition to the
1907                  * normal state.
1908                  */
1909                 adaprobedone(periph, NULL);
1910                 return(CAM_REQ_CMP);
1911         }
1912
1913         xpt_schedule(periph, CAM_PRIORITY_DEV);
1914
1915         return(CAM_REQ_CMP);
1916 }
1917
1918 static int
1919 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1920 {
1921         uint64_t lastlba = (uint64_t)-1;
1922         int c, lastcount = 0, off, ranges = 0;
1923
1924         bzero(req, sizeof(*req));
1925         TAILQ_INIT(&req->bps);
1926         do {
1927                 uint64_t lba = bp->bio_pblkno;
1928                 int count = bp->bio_bcount / softc->params.secsize;
1929
1930                 /* Try to extend the previous range. */
1931                 if (lba == lastlba) {
1932                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1933                         lastcount += c;
1934                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1935                         req->data[off + 6] = lastcount & 0xff;
1936                         req->data[off + 7] =
1937                                 (lastcount >> 8) & 0xff;
1938                         count -= c;
1939                         lba += c;
1940                 }
1941
1942                 while (count > 0) {
1943                         c = min(count, ATA_DSM_RANGE_MAX);
1944                         off = ranges * ATA_DSM_RANGE_SIZE;
1945                         req->data[off + 0] = lba & 0xff;
1946                         req->data[off + 1] = (lba >> 8) & 0xff;
1947                         req->data[off + 2] = (lba >> 16) & 0xff;
1948                         req->data[off + 3] = (lba >> 24) & 0xff;
1949                         req->data[off + 4] = (lba >> 32) & 0xff;
1950                         req->data[off + 5] = (lba >> 40) & 0xff;
1951                         req->data[off + 6] = c & 0xff;
1952                         req->data[off + 7] = (c >> 8) & 0xff;
1953                         lba += c;
1954                         count -= c;
1955                         lastcount = c;
1956                         ranges++;
1957                         /*
1958                          * Its the caller's responsibility to ensure the
1959                          * request will fit so we don't need to check for
1960                          * overrun here
1961                          */
1962                 }
1963                 lastlba = lba;
1964                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1965
1966                 bp = cam_iosched_next_trim(softc->cam_iosched);
1967                 if (bp == NULL)
1968                         break;
1969                 if (bp->bio_bcount / softc->params.secsize >
1970                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
1971                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
1972                         break;
1973                 }
1974         } while (1);
1975
1976         return (ranges);
1977 }
1978
1979 static void
1980 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1981 {
1982         struct trim_request *req = &softc->trim_req;
1983         int ranges;
1984
1985         ranges = ada_dsmtrim_req_create(softc, bp, req);
1986         cam_fill_ataio(ataio,
1987             ada_retry_count,
1988             adadone,
1989             CAM_DIR_OUT,
1990             0,
1991             req->data,
1992             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1993             ada_default_timeout * 1000);
1994         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1995             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
1996 }
1997
1998 static void
1999 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2000 {
2001         struct trim_request *req = &softc->trim_req;
2002         int ranges;
2003
2004         ranges = ada_dsmtrim_req_create(softc, bp, req);
2005         cam_fill_ataio(ataio,
2006             ada_retry_count,
2007             adadone,
2008             CAM_DIR_OUT,
2009             0,
2010             req->data,
2011             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
2012             ada_default_timeout * 1000);
2013         ata_ncq_cmd(ataio,
2014             ATA_SEND_FPDMA_QUEUED,
2015             0,
2016             howmany(ranges, ATA_DSM_BLK_RANGES));
2017         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
2018         ataio->ata_flags |= ATA_FLAG_AUX;
2019         ataio->aux = 1;
2020 }
2021
2022 static void
2023 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
2024 {
2025         struct trim_request *req = &softc->trim_req;
2026         uint64_t lba = bp->bio_pblkno;
2027         uint16_t count = bp->bio_bcount / softc->params.secsize;
2028
2029         bzero(req, sizeof(*req));
2030         TAILQ_INIT(&req->bps);
2031         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
2032
2033         cam_fill_ataio(ataio,
2034             ada_retry_count,
2035             adadone,
2036             CAM_DIR_NONE,
2037             0,
2038             NULL,
2039             0,
2040             ada_default_timeout*1000);
2041
2042         if (count >= 256)
2043                 count = 0;
2044         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2045 }
2046
2047 static int
2048 ada_zone_bio_to_ata(int disk_zone_cmd)
2049 {
2050         switch (disk_zone_cmd) {
2051         case DISK_ZONE_OPEN:
2052                 return ATA_ZM_OPEN_ZONE;
2053         case DISK_ZONE_CLOSE:
2054                 return ATA_ZM_CLOSE_ZONE;
2055         case DISK_ZONE_FINISH:
2056                 return ATA_ZM_FINISH_ZONE;
2057         case DISK_ZONE_RWP:
2058                 return ATA_ZM_RWP;
2059         }
2060
2061         return -1;
2062 }
2063
2064 static int
2065 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2066              int *queue_ccb)
2067 {
2068         struct ada_softc *softc;
2069         int error;
2070
2071         error = 0;
2072
2073         if (bp->bio_cmd != BIO_ZONE) {
2074                 error = EINVAL;
2075                 goto bailout;
2076         }
2077
2078         softc = periph->softc;
2079
2080         switch (bp->bio_zone.zone_cmd) {
2081         case DISK_ZONE_OPEN:
2082         case DISK_ZONE_CLOSE:
2083         case DISK_ZONE_FINISH:
2084         case DISK_ZONE_RWP: {
2085                 int zone_flags;
2086                 int zone_sa;
2087                 uint64_t lba;
2088
2089                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2090                 if (zone_sa == -1) {
2091                         xpt_print(periph->path, "Cannot translate zone "
2092                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2093                         error = EINVAL;
2094                         goto bailout;
2095                 }
2096
2097                 zone_flags = 0;
2098                 lba = bp->bio_zone.zone_params.rwp.id;
2099
2100                 if (bp->bio_zone.zone_params.rwp.flags &
2101                     DISK_ZONE_RWP_FLAG_ALL)
2102                         zone_flags |= ZBC_OUT_ALL;
2103
2104                 ata_zac_mgmt_out(&ccb->ataio,
2105                                  /*retries*/ ada_retry_count,
2106                                  /*cbfcnp*/ adadone,
2107                                  /*use_ncq*/ (softc->flags &
2108                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2109                                  /*zm_action*/ zone_sa,
2110                                  /*zone_id*/ lba,
2111                                  /*zone_flags*/ zone_flags,
2112                                  /*sector_count*/ 0,
2113                                  /*data_ptr*/ NULL,
2114                                  /*dxfer_len*/ 0,
2115                                  /*timeout*/ ada_default_timeout * 1000);
2116                 *queue_ccb = 1;
2117
2118                 break;
2119         }
2120         case DISK_ZONE_REPORT_ZONES: {
2121                 uint8_t *rz_ptr;
2122                 uint32_t num_entries, alloc_size;
2123                 struct disk_zone_report *rep;
2124
2125                 rep = &bp->bio_zone.zone_params.report;
2126
2127                 num_entries = rep->entries_allocated;
2128                 if (num_entries == 0) {
2129                         xpt_print(periph->path, "No entries allocated for "
2130                             "Report Zones request\n");
2131                         error = EINVAL;
2132                         goto bailout;
2133                 }
2134                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2135                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2136                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2137                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2138                 if (rz_ptr == NULL) {
2139                         xpt_print(periph->path, "Unable to allocate memory "
2140                            "for Report Zones request\n");
2141                         error = ENOMEM;
2142                         goto bailout;
2143                 }
2144
2145                 ata_zac_mgmt_in(&ccb->ataio,
2146                                 /*retries*/ ada_retry_count,
2147                                 /*cbcfnp*/ adadone,
2148                                 /*use_ncq*/ (softc->flags &
2149                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2150                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2151                                 /*zone_id*/ rep->starting_id,
2152                                 /*zone_flags*/ rep->rep_options,
2153                                 /*data_ptr*/ rz_ptr,
2154                                 /*dxfer_len*/ alloc_size,
2155                                 /*timeout*/ ada_default_timeout * 1000);
2156
2157                 /*
2158                  * For BIO_ZONE, this isn't normally needed.  However, it
2159                  * is used by devstat_end_transaction_bio() to determine
2160                  * how much data was transferred.
2161                  */
2162                 /*
2163                  * XXX KDM we have a problem.  But I'm not sure how to fix
2164                  * it.  devstat uses bio_bcount - bio_resid to calculate
2165                  * the amount of data transferred.   The GEOM disk code
2166                  * uses bio_length - bio_resid to calculate the amount of
2167                  * data in bio_completed.  We have different structure
2168                  * sizes above and below the ada(4) driver.  So, if we
2169                  * use the sizes above, the amount transferred won't be
2170                  * quite accurate for devstat.  If we use different sizes
2171                  * for bio_bcount and bio_length (above and below
2172                  * respectively), then the residual needs to match one or
2173                  * the other.  Everything is calculated after the bio
2174                  * leaves the driver, so changing the values around isn't
2175                  * really an option.  For now, just set the count to the
2176                  * passed in length.  This means that the calculations
2177                  * above (e.g. bio_completed) will be correct, but the
2178                  * amount of data reported to devstat will be slightly
2179                  * under or overstated.
2180                  */
2181                 bp->bio_bcount = bp->bio_length;
2182
2183                 *queue_ccb = 1;
2184
2185                 break;
2186         }
2187         case DISK_ZONE_GET_PARAMS: {
2188                 struct disk_zone_disk_params *params;
2189
2190                 params = &bp->bio_zone.zone_params.disk_params;
2191                 bzero(params, sizeof(*params));
2192
2193                 switch (softc->zone_mode) {
2194                 case ADA_ZONE_DRIVE_MANAGED:
2195                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2196                         break;
2197                 case ADA_ZONE_HOST_AWARE:
2198                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2199                         break;
2200                 case ADA_ZONE_HOST_MANAGED:
2201                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2202                         break;
2203                 default:
2204                 case ADA_ZONE_NONE:
2205                         params->zone_mode = DISK_ZONE_MODE_NONE;
2206                         break;
2207                 }
2208
2209                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2210                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2211
2212                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2213                         params->optimal_seq_zones = softc->optimal_seq_zones;
2214                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2215                 }
2216
2217                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2218                         params->optimal_nonseq_zones =
2219                             softc->optimal_nonseq_zones;
2220                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2221                 }
2222
2223                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2224                         params->max_seq_zones = softc->max_seq_zones;
2225                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2226                 }
2227                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2228                         params->flags |= DISK_ZONE_RZ_SUP;
2229
2230                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2231                         params->flags |= DISK_ZONE_OPEN_SUP;
2232
2233                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2234                         params->flags |= DISK_ZONE_CLOSE_SUP;
2235
2236                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2237                         params->flags |= DISK_ZONE_FINISH_SUP;
2238
2239                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2240                         params->flags |= DISK_ZONE_RWP_SUP;
2241                 break;
2242         }
2243         default:
2244                 break;
2245         }
2246 bailout:
2247         return (error);
2248 }
2249
2250 static void
2251 adastart(struct cam_periph *periph, union ccb *start_ccb)
2252 {
2253         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2254         struct ccb_ataio *ataio = &start_ccb->ataio;
2255
2256         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2257
2258         switch (softc->state) {
2259         case ADA_STATE_NORMAL:
2260         {
2261                 struct bio *bp;
2262                 u_int8_t tag_code;
2263
2264                 bp = cam_iosched_next_bio(softc->cam_iosched);
2265                 if (bp == NULL) {
2266                         xpt_release_ccb(start_ccb);
2267                         break;
2268                 }
2269
2270                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2271                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2272                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2273                         softc->flags |= ADA_FLAG_WAS_OTAG;
2274                         tag_code = 0;
2275                 } else {
2276                         tag_code = 1;
2277                 }
2278                 switch (bp->bio_cmd) {
2279                 case BIO_WRITE:
2280                 case BIO_READ:
2281                 {
2282                         uint64_t lba = bp->bio_pblkno;
2283                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2284                         void *data_ptr;
2285                         int rw_op;
2286
2287                         if (bp->bio_cmd == BIO_WRITE) {
2288                                 softc->flags |= ADA_FLAG_DIRTY;
2289                                 rw_op = CAM_DIR_OUT;
2290                         } else {
2291                                 rw_op = CAM_DIR_IN;
2292                         }
2293
2294                         data_ptr = bp->bio_data;
2295                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2296                                 rw_op |= CAM_DATA_BIO;
2297                                 data_ptr = bp;
2298                         }
2299
2300 #ifdef CAM_TEST_FAILURE
2301                         int fail = 0;
2302
2303                         /*
2304                          * Support the failure ioctls.  If the command is a
2305                          * read, and there are pending forced read errors, or
2306                          * if a write and pending write errors, then fail this
2307                          * operation with EIO.  This is useful for testing
2308                          * purposes.  Also, support having every Nth read fail.
2309                          *
2310                          * This is a rather blunt tool.
2311                          */
2312                         if (bp->bio_cmd == BIO_READ) {
2313                                 if (softc->force_read_error) {
2314                                         softc->force_read_error--;
2315                                         fail = 1;
2316                                 }
2317                                 if (softc->periodic_read_error > 0) {
2318                                         if (++softc->periodic_read_count >=
2319                                             softc->periodic_read_error) {
2320                                                 softc->periodic_read_count = 0;
2321                                                 fail = 1;
2322                                         }
2323                                 }
2324                         } else {
2325                                 if (softc->force_write_error) {
2326                                         softc->force_write_error--;
2327                                         fail = 1;
2328                                 }
2329                         }
2330                         if (fail) {
2331                                 biofinish(bp, NULL, EIO);
2332                                 xpt_release_ccb(start_ccb);
2333                                 adaschedule(periph);
2334                                 return;
2335                         }
2336 #endif
2337                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2338                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2339                             PAGE_SIZE == bp->bio_ma_n,
2340                             ("Short bio %p", bp));
2341                         cam_fill_ataio(ataio,
2342                             ada_retry_count,
2343                             adadone,
2344                             rw_op,
2345                             0,
2346                             data_ptr,
2347                             bp->bio_bcount,
2348                             ada_default_timeout*1000);
2349
2350                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2351                                 if (bp->bio_cmd == BIO_READ) {
2352                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2353                                             lba, count);
2354                                 } else {
2355                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2356                                             lba, count);
2357                                 }
2358                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2359                             (lba + count >= ATA_MAX_28BIT_LBA ||
2360                             count > 256)) {
2361                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2362                                         if (bp->bio_cmd == BIO_READ) {
2363                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2364                                                     0, lba, count);
2365                                         } else {
2366                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2367                                                     0, lba, count);
2368                                         }
2369                                 } else {
2370                                         if (bp->bio_cmd == BIO_READ) {
2371                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2372                                                     0, lba, count);
2373                                         } else {
2374                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2375                                                     0, lba, count);
2376                                         }
2377                                 }
2378                         } else {
2379                                 if (count == 256)
2380                                         count = 0;
2381                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2382                                         if (bp->bio_cmd == BIO_READ) {
2383                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2384                                                     0, lba, count);
2385                                         } else {
2386                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2387                                                     0, lba, count);
2388                                         }
2389                                 } else {
2390                                         if (bp->bio_cmd == BIO_READ) {
2391                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2392                                                     0, lba, count);
2393                                         } else {
2394                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2395                                                     0, lba, count);
2396                                         }
2397                                 }
2398                         }
2399                         break;
2400                 }
2401                 case BIO_DELETE:
2402                         switch (softc->delete_method) {
2403                         case ADA_DELETE_NCQ_DSM_TRIM:
2404                                 ada_ncq_dsmtrim(softc, bp, ataio);
2405                                 break;
2406                         case ADA_DELETE_DSM_TRIM:
2407                                 ada_dsmtrim(softc, bp, ataio);
2408                                 break;
2409                         case ADA_DELETE_CFA_ERASE:
2410                                 ada_cfaerase(softc, bp, ataio);
2411                                 break;
2412                         default:
2413                                 biofinish(bp, NULL, EOPNOTSUPP);
2414                                 xpt_release_ccb(start_ccb);
2415                                 adaschedule(periph);
2416                                 return;
2417                         }
2418                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2419                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2420                         cam_iosched_submit_trim(softc->cam_iosched);
2421                         goto out;
2422                 case BIO_FLUSH:
2423                         cam_fill_ataio(ataio,
2424                             1,
2425                             adadone,
2426                             CAM_DIR_NONE,
2427                             0,
2428                             NULL,
2429                             0,
2430                             ada_default_timeout*1000);
2431
2432                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2433                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2434                         else
2435                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2436                         break;
2437                 case BIO_ZONE: {
2438                         int error, queue_ccb;
2439
2440                         queue_ccb = 0;
2441
2442                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2443                         if ((error != 0)
2444                          || (queue_ccb == 0)) {
2445                                 biofinish(bp, NULL, error);
2446                                 xpt_release_ccb(start_ccb);
2447                                 return;
2448                         }
2449                         break;
2450                 }
2451                 }
2452                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2453                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2454 out:
2455                 start_ccb->ccb_h.ccb_bp = bp;
2456                 softc->outstanding_cmds++;
2457                 softc->refcount++;
2458                 cam_periph_unlock(periph);
2459                 xpt_action(start_ccb);
2460                 cam_periph_lock(periph);
2461
2462                 /* May have more work to do, so ensure we stay scheduled */
2463                 adaschedule(periph);
2464                 break;
2465         }
2466         case ADA_STATE_RAHEAD:
2467         case ADA_STATE_WCACHE:
2468         {
2469                 cam_fill_ataio(ataio,
2470                     1,
2471                     adadone,
2472                     CAM_DIR_NONE,
2473                     0,
2474                     NULL,
2475                     0,
2476                     ada_default_timeout*1000);
2477
2478                 if (softc->state == ADA_STATE_RAHEAD) {
2479                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2480                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2481                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2482                 } else {
2483                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2484                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2485                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2486                 }
2487                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2488                 xpt_action(start_ccb);
2489                 break;
2490         }
2491         case ADA_STATE_LOGDIR:
2492         {
2493                 struct ata_gp_log_dir *log_dir;
2494
2495                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2496                         adaprobedone(periph, start_ccb);
2497                         break;
2498                 }
2499
2500                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2501                 if (log_dir == NULL) {
2502                         xpt_print(periph->path, "Couldn't malloc log_dir "
2503                             "data\n");
2504                         softc->state = ADA_STATE_NORMAL;
2505                         xpt_release_ccb(start_ccb);
2506                         break;
2507                 }
2508
2509
2510                 ata_read_log(ataio,
2511                     /*retries*/1,
2512                     /*cbfcnp*/adadone,
2513                     /*log_address*/ ATA_LOG_DIRECTORY,
2514                     /*page_number*/ 0,
2515                     /*block_count*/ 1,
2516                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2517                                  CAM_ATAIO_DMA : 0,
2518                     /*data_ptr*/ (uint8_t *)log_dir,
2519                     /*dxfer_len*/sizeof(*log_dir),
2520                     /*timeout*/ada_default_timeout*1000);
2521
2522                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2523                 xpt_action(start_ccb);
2524                 break;
2525         }
2526         case ADA_STATE_IDDIR:
2527         {
2528                 struct ata_identify_log_pages *id_dir;
2529
2530                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2531                 if (id_dir == NULL) {
2532                         xpt_print(periph->path, "Couldn't malloc id_dir "
2533                             "data\n");
2534                         adaprobedone(periph, start_ccb);
2535                         break;
2536                 }
2537
2538                 ata_read_log(ataio,
2539                     /*retries*/1,
2540                     /*cbfcnp*/adadone,
2541                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2542                     /*page_number*/ ATA_IDL_PAGE_LIST,
2543                     /*block_count*/ 1,
2544                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2545                                  CAM_ATAIO_DMA : 0,
2546                     /*data_ptr*/ (uint8_t *)id_dir,
2547                     /*dxfer_len*/ sizeof(*id_dir),
2548                     /*timeout*/ada_default_timeout*1000);
2549
2550                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2551                 xpt_action(start_ccb);
2552                 break;
2553         }
2554         case ADA_STATE_SUP_CAP:
2555         {
2556                 struct ata_identify_log_sup_cap *sup_cap;
2557
2558                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2559                 if (sup_cap == NULL) {
2560                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2561                             "data\n");
2562                         adaprobedone(periph, start_ccb);
2563                         break;
2564                 }
2565
2566                 ata_read_log(ataio,
2567                     /*retries*/1,
2568                     /*cbfcnp*/adadone,
2569                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2570                     /*page_number*/ ATA_IDL_SUP_CAP,
2571                     /*block_count*/ 1,
2572                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2573                                  CAM_ATAIO_DMA : 0,
2574                     /*data_ptr*/ (uint8_t *)sup_cap,
2575                     /*dxfer_len*/ sizeof(*sup_cap),
2576                     /*timeout*/ada_default_timeout*1000);
2577
2578                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2579                 xpt_action(start_ccb);
2580                 break;
2581         }
2582         case ADA_STATE_ZONE:
2583         {
2584                 struct ata_zoned_info_log *ata_zone;
2585
2586                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2587                 if (ata_zone == NULL) {
2588                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2589                             "data\n");
2590                         adaprobedone(periph, start_ccb);
2591                         break;
2592                 }
2593
2594                 ata_read_log(ataio,
2595                     /*retries*/1,
2596                     /*cbfcnp*/adadone,
2597                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2598                     /*page_number*/ ATA_IDL_ZDI,
2599                     /*block_count*/ 1,
2600                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2601                                  CAM_ATAIO_DMA : 0,
2602                     /*data_ptr*/ (uint8_t *)ata_zone,
2603                     /*dxfer_len*/ sizeof(*ata_zone),
2604                     /*timeout*/ada_default_timeout*1000);
2605
2606                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2607                 xpt_action(start_ccb);
2608                 break;
2609         }
2610         }
2611 }
2612
2613 static void
2614 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2615 {
2616         struct ada_softc *softc;
2617
2618         softc = (struct ada_softc *)periph->softc;
2619
2620         if (ccb != NULL)
2621                 xpt_release_ccb(ccb);
2622
2623         softc->state = ADA_STATE_NORMAL;
2624         softc->flags |= ADA_FLAG_PROBED;
2625         adaschedule(periph);
2626         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2627                 softc->flags |= ADA_FLAG_ANNOUNCED;
2628                 cam_periph_unhold(periph);
2629         } else {
2630                 cam_periph_release_locked(periph);
2631         }
2632 }
2633
2634 static void
2635 adazonedone(struct cam_periph *periph, union ccb *ccb)
2636 {
2637         struct bio *bp;
2638
2639         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2640
2641         switch (bp->bio_zone.zone_cmd) {
2642         case DISK_ZONE_OPEN:
2643         case DISK_ZONE_CLOSE:
2644         case DISK_ZONE_FINISH:
2645         case DISK_ZONE_RWP:
2646                 break;
2647         case DISK_ZONE_REPORT_ZONES: {
2648                 uint32_t avail_len;
2649                 struct disk_zone_report *rep;
2650                 struct scsi_report_zones_hdr *hdr;
2651                 struct scsi_report_zones_desc *desc;
2652                 struct disk_zone_rep_entry *entry;
2653                 uint32_t hdr_len, num_avail;
2654                 uint32_t num_to_fill, i;
2655
2656                 rep = &bp->bio_zone.zone_params.report;
2657                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2658                 /*
2659                  * Note that bio_resid isn't normally used for zone
2660                  * commands, but it is used by devstat_end_transaction_bio()
2661                  * to determine how much data was transferred.  Because
2662                  * the size of the SCSI/ATA data structures is different
2663                  * than the size of the BIO interface structures, the
2664                  * amount of data actually transferred from the drive will
2665                  * be different than the amount of data transferred to
2666                  * the user.
2667                  */
2668                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2669                 if (avail_len < sizeof(*hdr)) {
2670                         /*
2671                          * Is there a better error than EIO here?  We asked
2672                          * for at least the header, and we got less than
2673                          * that.
2674                          */
2675                         bp->bio_error = EIO;
2676                         bp->bio_flags |= BIO_ERROR;
2677                         bp->bio_resid = bp->bio_bcount;
2678                         break;
2679                 }
2680
2681                 hdr_len = le32dec(hdr->length);
2682                 if (hdr_len > 0)
2683                         rep->entries_available = hdr_len / sizeof(*desc);
2684                 else
2685                         rep->entries_available = 0;
2686                 /*
2687                  * NOTE: using the same values for the BIO version of the
2688                  * same field as the SCSI/ATA values.  This means we could
2689                  * get some additional values that aren't defined in bio.h
2690                  * if more values of the same field are defined later.
2691                  */
2692                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2693                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2694                 /*
2695                  * If the drive reports no entries that match the query,
2696                  * we're done.
2697                  */
2698                 if (hdr_len == 0) {
2699                         rep->entries_filled = 0;
2700                         bp->bio_resid = bp->bio_bcount;
2701                         break;
2702                 }
2703
2704                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2705                                 hdr_len / sizeof(*desc));
2706                 /*
2707                  * If the drive didn't return any data, then we're done.
2708                  */
2709                 if (num_avail == 0) {
2710                         rep->entries_filled = 0;
2711                         bp->bio_resid = bp->bio_bcount;
2712                         break;
2713                 }
2714
2715                 num_to_fill = min(num_avail, rep->entries_allocated);
2716                 /*
2717                  * If the user didn't allocate any entries for us to fill,
2718                  * we're done.
2719                  */
2720                 if (num_to_fill == 0) {
2721                         rep->entries_filled = 0;
2722                         bp->bio_resid = bp->bio_bcount;
2723                         break;
2724                 }
2725
2726                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2727                      i < num_to_fill; i++, desc++, entry++) {
2728                         /*
2729                          * NOTE: we're mapping the values here directly
2730                          * from the SCSI/ATA bit definitions to the bio.h
2731                          * definitions.  There is also a warning in
2732                          * disk_zone.h, but the impact is that if
2733                          * additional values are added in the SCSI/ATA
2734                          * specs these will be visible to consumers of
2735                          * this interface.
2736                          */
2737                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2738                         entry->zone_condition =
2739                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2740                             SRZ_ZONE_COND_SHIFT;
2741                         entry->zone_flags |= desc->zone_flags &
2742                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2743                         entry->zone_length = le64dec(desc->zone_length);
2744                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2745                         entry->write_pointer_lba =
2746                             le64dec(desc->write_pointer_lba);
2747                 }
2748                 rep->entries_filled = num_to_fill;
2749                 /*
2750                  * Note that this residual is accurate from the user's
2751                  * standpoint, but the amount transferred isn't accurate
2752                  * from the standpoint of what actually came back from the
2753                  * drive.
2754                  */
2755                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2756                 break;
2757         }
2758         case DISK_ZONE_GET_PARAMS:
2759         default:
2760                 /*
2761                  * In theory we should not get a GET_PARAMS bio, since it
2762                  * should be handled without queueing the command to the
2763                  * drive.
2764                  */
2765                 panic("%s: Invalid zone command %d", __func__,
2766                     bp->bio_zone.zone_cmd);
2767                 break;
2768         }
2769
2770         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2771                 free(ccb->ataio.data_ptr, M_ATADA);
2772 }
2773
2774
2775 static void
2776 adadone(struct cam_periph *periph, union ccb *done_ccb)
2777 {
2778         struct ada_softc *softc;
2779         struct ccb_ataio *ataio;
2780         struct cam_path *path;
2781         uint32_t priority;
2782         int state;
2783
2784         softc = (struct ada_softc *)periph->softc;
2785         ataio = &done_ccb->ataio;
2786         path = done_ccb->ccb_h.path;
2787         priority = done_ccb->ccb_h.pinfo.priority;
2788
2789         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2790
2791         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2792         switch (state) {
2793         case ADA_CCB_BUFFER_IO:
2794         case ADA_CCB_TRIM:
2795         {
2796                 struct bio *bp;
2797                 int error;
2798
2799                 cam_periph_lock(periph);
2800                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2801                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2802                         error = adaerror(done_ccb, 0, 0);
2803                         if (error == ERESTART) {
2804                                 /* A retry was scheduled, so just return. */
2805                                 cam_periph_unlock(periph);
2806                                 return;
2807                         }
2808                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2809                                 cam_release_devq(path,
2810                                                  /*relsim_flags*/0,
2811                                                  /*reduction*/0,
2812                                                  /*timeout*/0,
2813                                                  /*getcount_only*/0);
2814                         /*
2815                          * If we get an error on an NCQ DSM TRIM, fall back
2816                          * to a non-NCQ DSM TRIM forever. Please note that if
2817                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2818                          * However, for this one trim, we treat it as advisory
2819                          * and return success up the stack.
2820                          */
2821                         if (state == ADA_CCB_TRIM &&
2822                             error != 0 &&
2823                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2824                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2825                                 error = 0;
2826                                 adasetdeletemethod(softc);
2827                         }
2828                 } else {
2829                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2830                                 panic("REQ_CMP with QFRZN");
2831
2832                         error = 0;
2833                 }
2834                 bp->bio_error = error;
2835                 if (error != 0) {
2836                         bp->bio_resid = bp->bio_bcount;
2837                         bp->bio_flags |= BIO_ERROR;
2838                 } else {
2839                         if (bp->bio_cmd == BIO_ZONE)
2840                                 adazonedone(periph, done_ccb);
2841                         else if (state == ADA_CCB_TRIM)
2842                                 bp->bio_resid = 0;
2843                         else
2844                                 bp->bio_resid = ataio->resid;
2845
2846                         if ((bp->bio_resid > 0)
2847                          && (bp->bio_cmd != BIO_ZONE))
2848                                 bp->bio_flags |= BIO_ERROR;
2849                 }
2850                 softc->outstanding_cmds--;
2851                 if (softc->outstanding_cmds == 0)
2852                         softc->flags |= ADA_FLAG_WAS_OTAG;
2853
2854                 /*
2855                  * We need to call cam_iosched before we call biodone so that we
2856                  * don't measure any activity that happens in the completion
2857                  * routine, which in the case of sendfile can be quite
2858                  * extensive.  Release the periph refcount taken in adastart()
2859                  * for each CCB.
2860                  */
2861                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2862                 xpt_release_ccb(done_ccb);
2863                 KASSERT(softc->refcount >= 1, ("adadone softc %p refcount %d", softc, softc->refcount));
2864                 softc->refcount--;
2865                 if (state == ADA_CCB_TRIM) {
2866                         TAILQ_HEAD(, bio) queue;
2867                         struct bio *bp1;
2868
2869                         TAILQ_INIT(&queue);
2870                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2871                         /*
2872                          * Normally, the xpt_release_ccb() above would make sure
2873                          * that when we have more work to do, that work would
2874                          * get kicked off. However, we specifically keep
2875                          * trim_running set to 0 before the call above to allow
2876                          * other I/O to progress when many BIO_DELETE requests
2877                          * are pushed down. We set trim_running to 0 and call
2878                          * daschedule again so that we don't stall if there are
2879                          * no other I/Os pending apart from BIO_DELETEs.
2880                          */
2881                         cam_iosched_trim_done(softc->cam_iosched);
2882                         adaschedule(periph);
2883                         cam_periph_unlock(periph);
2884                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2885                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2886                                 bp1->bio_error = error;
2887                                 if (error != 0) {
2888                                         bp1->bio_flags |= BIO_ERROR;
2889                                         bp1->bio_resid = bp1->bio_bcount;
2890                                 } else
2891                                         bp1->bio_resid = 0;
2892                                 biodone(bp1);
2893                         }
2894                 } else {
2895                         adaschedule(periph);
2896                         cam_periph_unlock(periph);
2897                         biodone(bp);
2898                 }
2899                 return;
2900         }
2901         case ADA_CCB_RAHEAD:
2902         {
2903                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2904                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2905                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2906                                 cam_release_devq(path, 0, 0, 0, FALSE);
2907                                 return;
2908                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2909                                 cam_release_devq(path,
2910                                     /*relsim_flags*/0,
2911                                     /*reduction*/0,
2912                                     /*timeout*/0,
2913                                     /*getcount_only*/0);
2914                         }
2915                 }
2916
2917                 /*
2918                  * Since our peripheral may be invalidated by an error
2919                  * above or an external event, we must release our CCB
2920                  * before releasing the reference on the peripheral.
2921                  * The peripheral will only go away once the last reference
2922                  * is removed, and we need it around for the CCB release
2923                  * operation.
2924                  */
2925
2926                 xpt_release_ccb(done_ccb);
2927                 softc->state = ADA_STATE_WCACHE;
2928                 xpt_schedule(periph, priority);
2929                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2930                 cam_release_devq(path, 0, 0, 0, FALSE);
2931                 return;
2932         }
2933         case ADA_CCB_WCACHE:
2934         {
2935                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2936                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2937                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2938                                 cam_release_devq(path, 0, 0, 0, FALSE);
2939                                 return;
2940                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2941                                 cam_release_devq(path,
2942                                     /*relsim_flags*/0,
2943                                     /*reduction*/0,
2944                                     /*timeout*/0,
2945                                     /*getcount_only*/0);
2946                         }
2947                 }
2948
2949                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2950                 cam_release_devq(path, 0, 0, 0, FALSE);
2951
2952                 if ((softc->flags & ADA_FLAG_CAN_LOG)
2953                  && (softc->zone_mode != ADA_ZONE_NONE)) {
2954                         xpt_release_ccb(done_ccb);
2955                         softc->state = ADA_STATE_LOGDIR;
2956                         xpt_schedule(periph, priority);
2957                 } else {
2958                         adaprobedone(periph, done_ccb);
2959                 }
2960                 return;
2961         }
2962         case ADA_CCB_LOGDIR:
2963         {
2964                 int error;
2965
2966                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2967                         error = 0;
2968                         softc->valid_logdir_len = 0;
2969                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
2970                         softc->valid_logdir_len =
2971                                 ataio->dxfer_len - ataio->resid;
2972                         if (softc->valid_logdir_len > 0)
2973                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
2974                                     min(softc->valid_logdir_len,
2975                                         sizeof(softc->ata_logdir)));
2976                         /*
2977                          * Figure out whether the Identify Device log is
2978                          * supported.  The General Purpose log directory
2979                          * has a header, and lists the number of pages
2980                          * available for each GP log identified by the
2981                          * offset into the list.
2982                          */
2983                         if ((softc->valid_logdir_len >=
2984                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
2985                          && (le16dec(softc->ata_logdir.header) ==
2986                              ATA_GP_LOG_DIR_VERSION)
2987                          && (le16dec(&softc->ata_logdir.num_pages[
2988                              (ATA_IDENTIFY_DATA_LOG *
2989                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
2990                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
2991                         } else {
2992                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
2993                         }
2994                 } else {
2995                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
2996                                          SF_RETRY_UA|SF_NO_PRINT);
2997                         if (error == ERESTART)
2998                                 return;
2999                         else if (error != 0) {
3000                                 /*
3001                                  * If we can't get the ATA log directory,
3002                                  * then ATA logs are effectively not
3003                                  * supported even if the bit is set in the
3004                                  * identify data.
3005                                  */
3006                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
3007                                                   ADA_FLAG_CAN_IDLOG);
3008                                 if ((done_ccb->ccb_h.status &
3009                                      CAM_DEV_QFRZN) != 0) {
3010                                         /* Don't wedge this device's queue */
3011                                         cam_release_devq(done_ccb->ccb_h.path,
3012                                                          /*relsim_flags*/0,
3013                                                          /*reduction*/0,
3014                                                          /*timeout*/0,
3015                                                          /*getcount_only*/0);
3016                                 }
3017                         }
3018
3019
3020                 }
3021
3022                 free(ataio->data_ptr, M_ATADA);
3023
3024                 if ((error == 0)
3025                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
3026                         softc->state = ADA_STATE_IDDIR;
3027                         xpt_release_ccb(done_ccb);
3028                         xpt_schedule(periph, priority);
3029                 } else
3030                         adaprobedone(periph, done_ccb);
3031
3032                 return;
3033         }
3034         case ADA_CCB_IDDIR: {
3035                 int error;
3036
3037                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3038                         off_t entries_offset, max_entries;
3039                         error = 0;
3040
3041                         softc->valid_iddir_len = 0;
3042                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
3043                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
3044                                           ADA_FLAG_CAN_ZONE);
3045                         softc->valid_iddir_len =
3046                                 ataio->dxfer_len - ataio->resid;
3047                         if (softc->valid_iddir_len > 0)
3048                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3049                                     min(softc->valid_iddir_len,
3050                                         sizeof(softc->ata_iddir)));
3051
3052                         entries_offset =
3053                             __offsetof(struct ata_identify_log_pages,entries);
3054                         max_entries = softc->valid_iddir_len - entries_offset;
3055                         if ((softc->valid_iddir_len > (entries_offset + 1))
3056                          && (le64dec(softc->ata_iddir.header) ==
3057                              ATA_IDLOG_REVISION)
3058                          && (softc->ata_iddir.entry_count > 0)) {
3059                                 int num_entries, i;
3060
3061                                 num_entries = softc->ata_iddir.entry_count;
3062                                 num_entries = min(num_entries,
3063                                    softc->valid_iddir_len - entries_offset);
3064                                 for (i = 0; i < num_entries &&
3065                                      i < max_entries; i++) {
3066                                         if (softc->ata_iddir.entries[i] ==
3067                                             ATA_IDL_SUP_CAP)
3068                                                 softc->flags |=
3069                                                     ADA_FLAG_CAN_SUPCAP;
3070                                         else if (softc->ata_iddir.entries[i]==
3071                                                  ATA_IDL_ZDI)
3072                                                 softc->flags |=
3073                                                     ADA_FLAG_CAN_ZONE;
3074
3075                                         if ((softc->flags &
3076                                              ADA_FLAG_CAN_SUPCAP)
3077                                          && (softc->flags &
3078                                              ADA_FLAG_CAN_ZONE))
3079                                                 break;
3080                                 }
3081                         }
3082                 } else {
3083                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3084                                          SF_RETRY_UA|SF_NO_PRINT);
3085                         if (error == ERESTART)
3086                                 return;
3087                         else if (error != 0) {
3088                                 /*
3089                                  * If we can't get the ATA Identify Data log
3090                                  * directory, then it effectively isn't
3091                                  * supported even if the ATA Log directory
3092                                  * a non-zero number of pages present for
3093                                  * this log.
3094                                  */
3095                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3096                                 if ((done_ccb->ccb_h.status &
3097                                      CAM_DEV_QFRZN) != 0) {
3098                                         /* Don't wedge this device's queue */
3099                                         cam_release_devq(done_ccb->ccb_h.path,
3100                                                          /*relsim_flags*/0,
3101                                                          /*reduction*/0,
3102                                                          /*timeout*/0,
3103                                                          /*getcount_only*/0);
3104                                 }
3105                         }
3106                 }
3107
3108                 free(ataio->data_ptr, M_ATADA);
3109
3110                 if ((error == 0)
3111                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3112                         softc->state = ADA_STATE_SUP_CAP;
3113                         xpt_release_ccb(done_ccb);
3114                         xpt_schedule(periph, priority);
3115                 } else
3116                         adaprobedone(periph, done_ccb);
3117                 return;
3118         }
3119         case ADA_CCB_SUP_CAP: {
3120                 int error;
3121
3122                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3123                         uint32_t valid_len;
3124                         size_t needed_size;
3125                         struct ata_identify_log_sup_cap *sup_cap;
3126                         error = 0;
3127
3128                         sup_cap = (struct ata_identify_log_sup_cap *)
3129                             ataio->data_ptr;
3130                         valid_len = ataio->dxfer_len - ataio->resid;
3131                         needed_size =
3132                             __offsetof(struct ata_identify_log_sup_cap,
3133                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3134                         if (valid_len >= needed_size) {
3135                                 uint64_t zoned, zac_cap;
3136
3137                                 zoned = le64dec(sup_cap->zoned_cap);
3138                                 if (zoned & ATA_ZONED_VALID) {
3139                                         /*
3140                                          * This should have already been
3141                                          * set, because this is also in the
3142                                          * ATA identify data.
3143                                          */
3144                                         if ((zoned & ATA_ZONED_MASK) ==
3145                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3146                                                 softc->zone_mode =
3147                                                     ADA_ZONE_HOST_AWARE;
3148                                         else if ((zoned & ATA_ZONED_MASK) ==
3149                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3150                                                 softc->zone_mode =
3151                                                     ADA_ZONE_DRIVE_MANAGED;
3152                                 }
3153
3154                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3155                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3156                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3157                                                 softc->zone_flags |=
3158                                                     ADA_ZONE_FLAG_RZ_SUP;
3159                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3160                                                 softc->zone_flags |=
3161                                                     ADA_ZONE_FLAG_OPEN_SUP;
3162                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3163                                                 softc->zone_flags |=
3164                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3165                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3166                                                 softc->zone_flags |=
3167                                                     ADA_ZONE_FLAG_FINISH_SUP;
3168                                         if (zac_cap & ATA_ND_RWP_SUP)
3169                                                 softc->zone_flags |=
3170                                                     ADA_ZONE_FLAG_RWP_SUP;
3171                                 } else {
3172                                         /*
3173                                          * This field was introduced in
3174                                          * ACS-4, r08 on April 28th, 2015.
3175                                          * If the drive firmware was written
3176                                          * to an earlier spec, it won't have
3177                                          * the field.  So, assume all
3178                                          * commands are supported.
3179                                          */
3180                                         softc->zone_flags |=
3181                                             ADA_ZONE_FLAG_SUP_MASK;
3182                                 }
3183
3184                         }
3185                 } else {
3186                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3187                                          SF_RETRY_UA|SF_NO_PRINT);
3188                         if (error == ERESTART)
3189                                 return;
3190                         else if (error != 0) {
3191                                 /*
3192                                  * If we can't get the ATA Identify Data
3193                                  * Supported Capabilities page, clear the
3194                                  * flag...
3195                                  */
3196                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3197                                 /*
3198                                  * And clear zone capabilities.
3199                                  */
3200                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3201                                 if ((done_ccb->ccb_h.status &
3202                                      CAM_DEV_QFRZN) != 0) {
3203                                         /* Don't wedge this device's queue */
3204                                         cam_release_devq(done_ccb->ccb_h.path,
3205                                                          /*relsim_flags*/0,
3206                                                          /*reduction*/0,
3207                                                          /*timeout*/0,
3208                                                          /*getcount_only*/0);
3209                                 }
3210                         }
3211                 }
3212
3213                 free(ataio->data_ptr, M_ATADA);
3214
3215                 if ((error == 0)
3216                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3217                         softc->state = ADA_STATE_ZONE;
3218                         xpt_release_ccb(done_ccb);
3219                         xpt_schedule(periph, priority);
3220                 } else
3221                         adaprobedone(periph, done_ccb);
3222                 return;
3223         }
3224         case ADA_CCB_ZONE: {
3225                 int error;
3226
3227                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3228                         struct ata_zoned_info_log *zi_log;
3229                         uint32_t valid_len;
3230                         size_t needed_size;
3231
3232                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3233
3234                         valid_len = ataio->dxfer_len - ataio->resid;
3235                         needed_size = __offsetof(struct ata_zoned_info_log,
3236                             version_info) + 1 + sizeof(zi_log->version_info);
3237                         if (valid_len >= needed_size) {
3238                                 uint64_t tmpvar;
3239
3240                                 tmpvar = le64dec(zi_log->zoned_cap);
3241                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3242                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3243                                                 softc->zone_flags |=
3244                                                     ADA_ZONE_FLAG_URSWRZ;
3245                                         else
3246                                                 softc->zone_flags &=
3247                                                     ~ADA_ZONE_FLAG_URSWRZ;
3248                                 }
3249                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3250                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3251                                         softc->zone_flags |=
3252                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3253                                         softc->optimal_seq_zones = (tmpvar &
3254                                             ATA_ZDI_OPT_SEQ_MASK);
3255                                 } else {
3256                                         softc->zone_flags &=
3257                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3258                                         softc->optimal_seq_zones = 0;
3259                                 }
3260
3261                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3262                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3263                                         softc->zone_flags |=
3264                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3265                                         softc->optimal_nonseq_zones =
3266                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3267                                 } else {
3268                                         softc->zone_flags &=
3269                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3270                                         softc->optimal_nonseq_zones = 0;
3271                                 }
3272
3273                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3274                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3275                                         softc->zone_flags |=
3276                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3277                                         softc->max_seq_zones =
3278                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3279                                 } else {
3280                                         softc->zone_flags &=
3281                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3282                                         softc->max_seq_zones = 0;
3283                                 }
3284                         }
3285                 } else {
3286                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3287                                          SF_RETRY_UA|SF_NO_PRINT);
3288                         if (error == ERESTART)
3289                                 return;
3290                         else if (error != 0) {
3291                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3292                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3293
3294                                 if ((done_ccb->ccb_h.status &
3295                                      CAM_DEV_QFRZN) != 0) {
3296                                         /* Don't wedge this device's queue */
3297                                         cam_release_devq(done_ccb->ccb_h.path,
3298                                                          /*relsim_flags*/0,
3299                                                          /*reduction*/0,
3300                                                          /*timeout*/0,
3301                                                          /*getcount_only*/0);
3302                                 }
3303                         }
3304
3305                 }
3306                 free(ataio->data_ptr, M_ATADA);
3307
3308                 adaprobedone(periph, done_ccb);
3309                 return;
3310         }
3311         case ADA_CCB_DUMP:
3312                 /* No-op.  We're polling */
3313                 return;
3314         default:
3315                 break;
3316         }
3317         xpt_release_ccb(done_ccb);
3318 }
3319
3320 static int
3321 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3322 {
3323 #ifdef CAM_IO_STATS
3324         struct ada_softc *softc;
3325         struct cam_periph *periph;
3326
3327         periph = xpt_path_periph(ccb->ccb_h.path);
3328         softc = (struct ada_softc *)periph->softc;
3329
3330         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3331         case CAM_CMD_TIMEOUT:
3332                 softc->timeouts++;
3333                 break;
3334         case CAM_REQ_ABORTED:
3335         case CAM_REQ_CMP_ERR:
3336         case CAM_REQ_TERMIO:
3337         case CAM_UNREC_HBA_ERROR:
3338         case CAM_DATA_RUN_ERR:
3339         case CAM_ATA_STATUS_ERROR:
3340                 softc->errors++;
3341                 break;
3342         default:
3343                 break;
3344         }
3345 #endif
3346
3347         return(cam_periph_error(ccb, cam_flags, sense_flags));
3348 }
3349
3350 static void
3351 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
3352 {
3353         struct ada_softc *softc = (struct ada_softc *)periph->softc;
3354         struct disk_params *dp = &softc->params;
3355         u_int64_t lbasize48;
3356         u_int32_t lbasize;
3357
3358         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3359         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3360                 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
3361                 dp->heads = cgd->ident_data.current_heads;
3362                 dp->secs_per_track = cgd->ident_data.current_sectors;
3363                 dp->cylinders = cgd->ident_data.cylinders;
3364                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3365                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3366         } else {
3367                 dp->heads = cgd->ident_data.heads;
3368                 dp->secs_per_track = cgd->ident_data.sectors;
3369                 dp->cylinders = cgd->ident_data.cylinders;
3370                 dp->sectors = cgd->ident_data.cylinders *
3371                               (u_int32_t)(dp->heads * dp->secs_per_track);
3372         }
3373         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3374                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3375
3376         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3377         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3378                 dp->sectors = lbasize;
3379
3380         /* use the 48bit LBA size if valid */
3381         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3382                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3383                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3384                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3385         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3386             lbasize48 > ATA_MAX_28BIT_LBA)
3387                 dp->sectors = lbasize48;
3388 }
3389
3390 static void
3391 adasendorderedtag(void *arg)
3392 {
3393         struct ada_softc *softc = arg;
3394
3395         if (ada_send_ordered) {
3396                 if (softc->outstanding_cmds > 0) {
3397                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3398                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3399                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3400                 }
3401         }
3402         /* Queue us up again */
3403         callout_reset(&softc->sendordered_c,
3404             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3405             adasendorderedtag, softc);
3406 }
3407
3408 /*
3409  * Step through all ADA peripheral drivers, and if the device is still open,
3410  * sync the disk cache to physical media.
3411  */
3412 static void
3413 adaflush(void)
3414 {
3415         struct cam_periph *periph;
3416         struct ada_softc *softc;
3417         union ccb *ccb;
3418         int error;
3419
3420         CAM_PERIPH_FOREACH(periph, &adadriver) {
3421                 softc = (struct ada_softc *)periph->softc;
3422                 if (SCHEDULER_STOPPED()) {
3423                         /* If we paniced with the lock held, do not recurse. */
3424                         if (!cam_periph_owned(periph) &&
3425                             (softc->flags & ADA_FLAG_OPEN)) {
3426                                 adadump(softc->disk, NULL, 0, 0, 0);
3427                         }
3428                         continue;
3429                 }
3430                 cam_periph_lock(periph);
3431                 /*
3432                  * We only sync the cache if the drive is still open, and
3433                  * if the drive is capable of it..
3434                  */
3435                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3436                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3437                         cam_periph_unlock(periph);
3438                         continue;
3439                 }
3440
3441                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3442                 cam_fill_ataio(&ccb->ataio,
3443                                     0,
3444                                     NULL,
3445                                     CAM_DIR_NONE,
3446                                     0,
3447                                     NULL,
3448                                     0,
3449                                     ada_default_timeout*1000);
3450                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3451                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3452                 else
3453                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3454
3455                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3456                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3457                     softc->disk->d_devstat);
3458                 if (error != 0)
3459                         xpt_print(periph->path, "Synchronize cache failed\n");
3460                 xpt_release_ccb(ccb);
3461                 cam_periph_unlock(periph);
3462         }
3463 }
3464
3465 static void
3466 adaspindown(uint8_t cmd, int flags)
3467 {
3468         struct cam_periph *periph;
3469         struct ada_softc *softc;
3470         struct ccb_ataio local_ccb;
3471         int error;
3472
3473         CAM_PERIPH_FOREACH(periph, &adadriver) {
3474                 /* If we paniced with lock held - not recurse here. */
3475                 if (cam_periph_owned(periph))
3476                         continue;
3477                 cam_periph_lock(periph);
3478                 softc = (struct ada_softc *)periph->softc;
3479                 /*
3480                  * We only spin-down the drive if it is capable of it..
3481                  */
3482                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3483                         cam_periph_unlock(periph);
3484                         continue;
3485                 }
3486
3487                 if (bootverbose)
3488                         xpt_print(periph->path, "spin-down\n");
3489
3490                 memset(&local_ccb, 0, sizeof(local_ccb));
3491                 xpt_setup_ccb(&local_ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
3492                 local_ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
3493
3494                 cam_fill_ataio(&local_ccb,
3495                                     0,
3496                                     NULL,
3497                                     CAM_DIR_NONE | flags,
3498                                     0,
3499                                     NULL,
3500                                     0,
3501                                     ada_default_timeout*1000);
3502                 ata_28bit_cmd(&local_ccb, cmd, 0, 0, 0);
3503                 error = cam_periph_runccb((union ccb *)&local_ccb, adaerror,
3504                     /*cam_flags*/0, /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3505                     softc->disk->d_devstat);
3506                 if (error != 0)
3507                         xpt_print(periph->path, "Spin-down disk failed\n");
3508                 cam_periph_unlock(periph);
3509         }
3510 }
3511
3512 static void
3513 adashutdown(void *arg, int howto)
3514 {
3515         int how;
3516
3517         adaflush();
3518
3519         /*
3520          * STANDBY IMMEDIATE saves any volatile data to the drive. It also spins
3521          * down hard drives. IDLE IMMEDIATE also saves the volatile data without
3522          * a spindown. We send the former when we expect to lose power soon. For
3523          * a warm boot, we send the latter to avoid a thundering herd of spinups
3524          * just after the kernel loads while probing. We have to do something to
3525          * flush the data because the BIOS in many systems resets the HBA
3526          * causing a COMINIT/COMRESET negotiation, which some drives interpret
3527          * as license to toss the volatile data, and others count as unclean
3528          * shutdown when in the Active PM state in SMART attributes.
3529          *
3530          * adaspindown will ensure that we don't send this to a drive that
3531          * doesn't support it.
3532          */
3533         if (ada_spindown_shutdown != 0) {
3534                 how = (howto & (RB_HALT | RB_POWEROFF | RB_POWERCYCLE)) ?
3535                     ATA_STANDBY_IMMEDIATE : ATA_IDLE_IMMEDIATE;
3536                 adaspindown(how, 0);
3537         }
3538 }
3539
3540 static void
3541 adasuspend(void *arg)
3542 {
3543
3544         adaflush();
3545         /*
3546          * SLEEP also fushes any volatile data, like STANDBY IMEDIATE,
3547          * so we don't need to send it as well.
3548          */
3549         if (ada_spindown_suspend != 0)
3550                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3551 }
3552
3553 static void
3554 adaresume(void *arg)
3555 {
3556         struct cam_periph *periph;
3557         struct ada_softc *softc;
3558
3559         if (ada_spindown_suspend == 0)
3560                 return;
3561
3562         CAM_PERIPH_FOREACH(periph, &adadriver) {
3563                 cam_periph_lock(periph);
3564                 softc = (struct ada_softc *)periph->softc;
3565                 /*
3566                  * We only spin-down the drive if it is capable of it..
3567                  */
3568                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3569                         cam_periph_unlock(periph);
3570                         continue;
3571                 }
3572
3573                 if (bootverbose)
3574                         xpt_print(periph->path, "resume\n");
3575
3576                 /*
3577                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3578                  * sleep request.
3579                  */
3580                 cam_release_devq(periph->path,
3581                          /*relsim_flags*/0,
3582                          /*openings*/0,
3583                          /*timeout*/0,
3584                          /*getcount_only*/0);
3585
3586                 cam_periph_unlock(periph);
3587         }
3588 }
3589
3590 #endif /* _KERNEL */