sys/cam/ata/ata_da.c

   1 /*-
   2  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification, immediately at the beginning of the file.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include "opt_ada.h"
  31
  32 #include <sys/param.h>
  33
  34 #ifdef _KERNEL
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/bio.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/taskqueue.h>
  40 #include <sys/lock.h>
  41 #include <sys/mutex.h>
  42 #include <sys/conf.h>
  43 #include <sys/devicestat.h>
  44 #include <sys/eventhandler.h>
  45 #include <sys/malloc.h>
  46 #include <sys/endian.h>
  47 #include <sys/cons.h>
  48 #include <sys/proc.h>
  49 #include <sys/reboot.h>
  50 #include <sys/sbuf.h>
  51 #include <geom/geom_disk.h>
  52 #endif /* _KERNEL */
  53
  54 #ifndef _KERNEL
  55 #include <stdio.h>
  56 #include <string.h>
  57 #endif /* _KERNEL */
  58
  59 #include <cam/cam.h>
  60 #include <cam/cam_ccb.h>
  61 #include <cam/cam_periph.h>
  62 #include <cam/cam_xpt_periph.h>
  63 #include <cam/scsi/scsi_all.h>
  64 #include <cam/scsi/scsi_da.h>
  65 #include <cam/cam_sim.h>
  66 #include <cam/cam_iosched.h>
  67
  68 #include <cam/ata/ata_all.h>
  69
  70 #include <machine/md_var.h>     /* geometry translation */
  71
  72 #ifdef _KERNEL
  73
  74 #define ATA_MAX_28BIT_LBA               268435455UL
  75
  76 extern int iosched_debug;
  77
  78 typedef enum {
  79         ADA_STATE_RAHEAD,
  80         ADA_STATE_WCACHE,
  81         ADA_STATE_LOGDIR,
  82         ADA_STATE_IDDIR,
  83         ADA_STATE_SUP_CAP,
  84         ADA_STATE_ZONE,
  85         ADA_STATE_NORMAL
  86 } ada_state;
  87
  88 typedef enum {
  89         ADA_FLAG_CAN_48BIT      = 0x00000002,
  90         ADA_FLAG_CAN_FLUSHCACHE = 0x00000004,
  91         ADA_FLAG_CAN_NCQ        = 0x00000008,
  92         ADA_FLAG_CAN_DMA        = 0x00000010,
  93         ADA_FLAG_NEED_OTAG      = 0x00000020,
  94         ADA_FLAG_WAS_OTAG       = 0x00000040,
  95         ADA_FLAG_CAN_TRIM       = 0x00000080,
  96         ADA_FLAG_OPEN           = 0x00000100,
  97         ADA_FLAG_SCTX_INIT      = 0x00000200,
  98         ADA_FLAG_CAN_CFA        = 0x00000400,
  99         ADA_FLAG_CAN_POWERMGT   = 0x00000800,
 100         ADA_FLAG_CAN_DMA48      = 0x00001000,
 101         ADA_FLAG_CAN_LOG        = 0x00002000,
 102         ADA_FLAG_CAN_IDLOG      = 0x00004000,
 103         ADA_FLAG_CAN_SUPCAP     = 0x00008000,
 104         ADA_FLAG_CAN_ZONE       = 0x00010000,
 105         ADA_FLAG_CAN_WCACHE     = 0x00020000,
 106         ADA_FLAG_CAN_RAHEAD     = 0x00040000,
 107         ADA_FLAG_PROBED         = 0x00080000,
 108         ADA_FLAG_ANNOUNCED      = 0x00100000,
 109         ADA_FLAG_DIRTY          = 0x00200000,
 110         ADA_FLAG_CAN_NCQ_TRIM   = 0x00400000,   /* CAN_TRIM also set */
 111         ADA_FLAG_PIM_ATA_EXT    = 0x00800000
 112 } ada_flags;
 113
 114 typedef enum {
 115         ADA_Q_NONE              = 0x00,
 116         ADA_Q_4K                = 0x01,
 117         ADA_Q_NCQ_TRIM_BROKEN   = 0x02,
 118         ADA_Q_LOG_BROKEN        = 0x04,
 119         ADA_Q_SMR_DM            = 0x08
 120 } ada_quirks;
 121
 122 #define ADA_Q_BIT_STRING        \
 123         "\020"                  \
 124         "\0014K"                \
 125         "\002NCQ_TRIM_BROKEN"   \
 126         "\003LOG_BROKEN"        \
 127         "\004SMR_DM"
 128
 129 typedef enum {
 130         ADA_CCB_RAHEAD          = 0x01,
 131         ADA_CCB_WCACHE          = 0x02,
 132         ADA_CCB_BUFFER_IO       = 0x03,
 133         ADA_CCB_DUMP            = 0x05,
 134         ADA_CCB_TRIM            = 0x06,
 135         ADA_CCB_LOGDIR          = 0x07,
 136         ADA_CCB_IDDIR           = 0x08,
 137         ADA_CCB_SUP_CAP         = 0x09,
 138         ADA_CCB_ZONE            = 0x0a,
 139         ADA_CCB_TYPE_MASK       = 0x0F,
 140 } ada_ccb_state;
 141
 142 typedef enum {
 143         ADA_ZONE_NONE           = 0x00,
 144         ADA_ZONE_DRIVE_MANAGED  = 0x01,
 145         ADA_ZONE_HOST_AWARE     = 0x02,
 146         ADA_ZONE_HOST_MANAGED   = 0x03
 147 } ada_zone_mode;
 148
 149 typedef enum {
 150         ADA_ZONE_FLAG_RZ_SUP            = 0x0001,
 151         ADA_ZONE_FLAG_OPEN_SUP          = 0x0002,
 152         ADA_ZONE_FLAG_CLOSE_SUP         = 0x0004,
 153         ADA_ZONE_FLAG_FINISH_SUP        = 0x0008,
 154         ADA_ZONE_FLAG_RWP_SUP           = 0x0010,
 155         ADA_ZONE_FLAG_SUP_MASK          = (ADA_ZONE_FLAG_RZ_SUP |
 156                                            ADA_ZONE_FLAG_OPEN_SUP |
 157                                            ADA_ZONE_FLAG_CLOSE_SUP |
 158                                            ADA_ZONE_FLAG_FINISH_SUP |
 159                                            ADA_ZONE_FLAG_RWP_SUP),
 160         ADA_ZONE_FLAG_URSWRZ            = 0x0020,
 161         ADA_ZONE_FLAG_OPT_SEQ_SET       = 0x0040,
 162         ADA_ZONE_FLAG_OPT_NONSEQ_SET    = 0x0080,
 163         ADA_ZONE_FLAG_MAX_SEQ_SET       = 0x0100,
 164         ADA_ZONE_FLAG_SET_MASK          = (ADA_ZONE_FLAG_OPT_SEQ_SET |
 165                                            ADA_ZONE_FLAG_OPT_NONSEQ_SET |
 166                                            ADA_ZONE_FLAG_MAX_SEQ_SET)
 167 } ada_zone_flags;
 168
 169 static struct ada_zone_desc {
 170         ada_zone_flags value;
 171         const char *desc;
 172 } ada_zone_desc_table[] = {
 173         {ADA_ZONE_FLAG_RZ_SUP, "Report Zones" },
 174         {ADA_ZONE_FLAG_OPEN_SUP, "Open" },
 175         {ADA_ZONE_FLAG_CLOSE_SUP, "Close" },
 176         {ADA_ZONE_FLAG_FINISH_SUP, "Finish" },
 177         {ADA_ZONE_FLAG_RWP_SUP, "Reset Write Pointer" },
 178 };
 179
 180
 181 /* Offsets into our private area for storing information */
 182 #define ccb_state       ppriv_field0
 183 #define ccb_bp          ppriv_ptr1
 184
 185 typedef enum {
 186         ADA_DELETE_NONE,
 187         ADA_DELETE_DISABLE,
 188         ADA_DELETE_CFA_ERASE,
 189         ADA_DELETE_DSM_TRIM,
 190         ADA_DELETE_NCQ_DSM_TRIM,
 191         ADA_DELETE_MIN = ADA_DELETE_CFA_ERASE,
 192         ADA_DELETE_MAX = ADA_DELETE_NCQ_DSM_TRIM,
 193 } ada_delete_methods;
 194
 195 static const char *ada_delete_method_names[] =
 196     { "NONE", "DISABLE", "CFA_ERASE", "DSM_TRIM", "NCQ_DSM_TRIM" };
 197 #if 0
 198 static const char *ada_delete_method_desc[] =
 199     { "NONE", "DISABLED", "CFA Erase", "DSM Trim", "DSM Trim via NCQ" };
 200 #endif
 201
 202 struct disk_params {
 203         u_int8_t  heads;
 204         u_int8_t  secs_per_track;
 205         u_int32_t cylinders;
 206         u_int32_t secsize;      /* Number of bytes/logical sector */
 207         u_int64_t sectors;      /* Total number sectors */
 208 };
 209
 210 #define TRIM_MAX_BLOCKS 8
 211 #define TRIM_MAX_RANGES (TRIM_MAX_BLOCKS * ATA_DSM_BLK_RANGES)
 212 struct trim_request {
 213         uint8_t         data[TRIM_MAX_RANGES * ATA_DSM_RANGE_SIZE];
 214         TAILQ_HEAD(, bio) bps;
 215 };
 216
 217 struct ada_softc {
 218         struct   cam_iosched_softc *cam_iosched;
 219         int      outstanding_cmds;      /* Number of active commands */
 220         int      refcount;              /* Active xpt_action() calls */
 221         ada_state state;
 222         ada_flags flags;
 223         ada_zone_mode zone_mode;
 224         ada_zone_flags zone_flags;
 225         struct ata_gp_log_dir ata_logdir;
 226         int valid_logdir_len;
 227         struct ata_identify_log_pages ata_iddir;
 228         int valid_iddir_len;
 229         uint64_t optimal_seq_zones;
 230         uint64_t optimal_nonseq_zones;
 231         uint64_t max_seq_zones;
 232         ada_quirks quirks;
 233         ada_delete_methods delete_method;
 234         int      trim_max_ranges;
 235         int      read_ahead;
 236         int      write_cache;
 237         int      unmappedio;
 238         int      rotating;
 239 #ifdef ADA_TEST_FAILURE
 240         int      force_read_error;
 241         int      force_write_error;
 242         int      periodic_read_error;
 243         int      periodic_read_count;
 244 #endif
 245         struct   disk_params params;
 246         struct   disk *disk;
 247         struct task             sysctl_task;
 248         struct sysctl_ctx_list  sysctl_ctx;
 249         struct sysctl_oid       *sysctl_tree;
 250         struct callout          sendordered_c;
 251         struct trim_request     trim_req;
 252 #ifdef CAM_IO_STATS
 253         struct sysctl_ctx_list  sysctl_stats_ctx;
 254         struct sysctl_oid       *sysctl_stats_tree;
 255         u_int   timeouts;
 256         u_int   errors;
 257         u_int   invalidations;
 258 #endif
 259 };
 260
 261 struct ada_quirk_entry {
 262         struct scsi_inquiry_pattern inq_pat;
 263         ada_quirks quirks;
 264 };
 265
 266 static struct ada_quirk_entry ada_quirk_table[] =
 267 {
 268         {
 269                 /* Hitachi Advanced Format (4k) drives */
 270                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Hitachi H??????????E3*", "*" },
 271                 /*quirks*/ADA_Q_4K
 272         },
 273         {
 274                 /* Samsung Advanced Format (4k) drives */
 275                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD155UI*", "*" },
 276                 /*quirks*/ADA_Q_4K
 277         },
 278         {
 279                 /* Samsung Advanced Format (4k) drives */
 280                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD204UI*", "*" },
 281                 /*quirks*/ADA_Q_4K
 282         },
 283         {
 284                 /* Seagate Barracuda Green Advanced Format (4k) drives */
 285                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DL*", "*" },
 286                 /*quirks*/ADA_Q_4K
 287         },
 288         {
 289                 /* Seagate Barracuda Advanced Format (4k) drives */
 290                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???DM*", "*" },
 291                 /*quirks*/ADA_Q_4K
 292         },
 293         {
 294                 /* Seagate Barracuda Advanced Format (4k) drives */
 295                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST????DM*", "*" },
 296                 /*quirks*/ADA_Q_4K
 297         },
 298         {
 299                 /* Seagate Momentus Advanced Format (4k) drives */
 300                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500423AS*", "*" },
 301                 /*quirks*/ADA_Q_4K
 302         },
 303         {
 304                 /* Seagate Momentus Advanced Format (4k) drives */
 305                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9500424AS*", "*" },
 306                 /*quirks*/ADA_Q_4K
 307         },
 308         {
 309                 /* Seagate Momentus Advanced Format (4k) drives */
 310                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640423AS*", "*" },
 311                 /*quirks*/ADA_Q_4K
 312         },
 313         {
 314                 /* Seagate Momentus Advanced Format (4k) drives */
 315                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9640424AS*", "*" },
 316                 /*quirks*/ADA_Q_4K
 317         },
 318         {
 319                 /* Seagate Momentus Advanced Format (4k) drives */
 320                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750420AS*", "*" },
 321                 /*quirks*/ADA_Q_4K
 322         },
 323         {
 324                 /* Seagate Momentus Advanced Format (4k) drives */
 325                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750422AS*", "*" },
 326                 /*quirks*/ADA_Q_4K
 327         },
 328         {
 329                 /* Seagate Momentus Advanced Format (4k) drives */
 330                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST9750423AS*", "*" },
 331                 /*quirks*/ADA_Q_4K
 332         },
 333         {
 334                 /* Seagate Momentus Thin Advanced Format (4k) drives */
 335                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST???LT*", "*" },
 336                 /*quirks*/ADA_Q_4K
 337         },
 338         {
 339                 /* WDC Caviar Red Advanced Format (4k) drives */
 340                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????CX*", "*" },
 341                 /*quirks*/ADA_Q_4K
 342         },
 343         {
 344                 /* WDC Caviar Green Advanced Format (4k) drives */
 345                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RS*", "*" },
 346                 /*quirks*/ADA_Q_4K
 347         },
 348         {
 349                 /* WDC Caviar Green/Red Advanced Format (4k) drives */
 350                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD????RX*", "*" },
 351                 /*quirks*/ADA_Q_4K
 352         },
 353         {
 354                 /* WDC Caviar Red Advanced Format (4k) drives */
 355                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????CX*", "*" },
 356                 /*quirks*/ADA_Q_4K
 357         },
 358         {
 359                 /* WDC Caviar Black Advanced Format (4k) drives */
 360                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????EX*", "*" },
 361                 /*quirks*/ADA_Q_4K
 362         },
 363         {
 364                 /* WDC Caviar Green Advanced Format (4k) drives */
 365                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RS*", "*" },
 366                 /*quirks*/ADA_Q_4K
 367         },
 368         {
 369                 /* WDC Caviar Green Advanced Format (4k) drives */
 370                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD??????RX*", "*" },
 371                 /*quirks*/ADA_Q_4K
 372         },
 373         {
 374                 /* WDC Scorpio Black Advanced Format (4k) drives */
 375                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PKT*", "*" },
 376                 /*quirks*/ADA_Q_4K
 377         },
 378         {
 379                 /* WDC Scorpio Black Advanced Format (4k) drives */
 380                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PKT*", "*" },
 381                 /*quirks*/ADA_Q_4K
 382         },
 383         {
 384                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 385                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD???PVT*", "*" },
 386                 /*quirks*/ADA_Q_4K
 387         },
 388         {
 389                 /* WDC Scorpio Blue Advanced Format (4k) drives */
 390                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "WDC WD?????PVT*", "*" },
 391                 /*quirks*/ADA_Q_4K
 392         },
 393         /* SSDs */
 394         {
 395                 /*
 396                  * Corsair Force 2 SSDs
 397                  * 4k optimised & trim only works in 4k requests + 4k aligned
 398                  */
 399                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair CSSD-F*", "*" },
 400                 /*quirks*/ADA_Q_4K
 401         },
 402         {
 403                 /*
 404                  * Corsair Force 3 SSDs
 405                  * 4k optimised & trim only works in 4k requests + 4k aligned
 406                  */
 407                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force 3*", "*" },
 408                 /*quirks*/ADA_Q_4K
 409         },
 410         {
 411                 /*
 412                  * Corsair Neutron GTX SSDs
 413                  * 4k optimised & trim only works in 4k requests + 4k aligned
 414                  */
 415                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Neutron GTX*", "*" },
 416                 /*quirks*/ADA_Q_4K
 417         },
 418         {
 419                 /*
 420                  * Corsair Force GT & GS SSDs
 421                  * 4k optimised & trim only works in 4k requests + 4k aligned
 422                  */
 423                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Corsair Force G*", "*" },
 424                 /*quirks*/ADA_Q_4K
 425         },
 426         {
 427                 /*
 428                  * Crucial M4 SSDs
 429                  * 4k optimised & trim only works in 4k requests + 4k aligned
 430                  */
 431                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "M4-CT???M4SSD2*", "*" },
 432                 /*quirks*/ADA_Q_4K
 433         },
 434         {
 435                 /*
 436                  * Crucial M500 SSDs MU07 firmware
 437                  * NCQ Trim works
 438                  */
 439                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "MU07" },
 440                 /*quirks*/0
 441         },
 442         {
 443                 /*
 444                  * Crucial M500 SSDs all other firmware
 445                  * NCQ Trim doesn't work
 446                  */
 447                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M500*", "*" },
 448                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 449         },
 450         {
 451                 /*
 452                  * Crucial M550 SSDs
 453                  * NCQ Trim doesn't work, but only on MU01 firmware
 454                  */
 455                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*M550*", "MU01" },
 456                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 457         },
 458         {
 459                 /*
 460                  * Crucial MX100 SSDs
 461                  * NCQ Trim doesn't work, but only on MU01 firmware
 462                  */
 463                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Crucial CT*MX100*", "MU01" },
 464                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 465         },
 466         {
 467                 /*
 468                  * Crucial RealSSD C300 SSDs
 469                  * 4k optimised
 470                  */
 471                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "C300-CTFDDAC???MAG*",
 472                 "*" }, /*quirks*/ADA_Q_4K
 473         },
 474         {
 475                 /*
 476                  * FCCT M500 SSDs
 477                  * NCQ Trim doesn't work
 478                  */
 479                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FCCT*M500*", "*" },
 480                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 481         },
 482         {
 483                 /*
 484                  * Intel 320 Series SSDs
 485                  * 4k optimised & trim only works in 4k requests + 4k aligned
 486                  */
 487                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2CW*", "*" },
 488                 /*quirks*/ADA_Q_4K
 489         },
 490         {
 491                 /*
 492                  * Intel 330 Series SSDs
 493                  * 4k optimised & trim only works in 4k requests + 4k aligned
 494                  */
 495                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2CT*", "*" },
 496                 /*quirks*/ADA_Q_4K
 497         },
 498         {
 499                 /*
 500                  * Intel 510 Series SSDs
 501                  * 4k optimised & trim only works in 4k requests + 4k aligned
 502                  */
 503                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2MH*", "*" },
 504                 /*quirks*/ADA_Q_4K
 505         },
 506         {
 507                 /*
 508                  * Intel 520 Series SSDs
 509                  * 4k optimised & trim only works in 4k requests + 4k aligned
 510                  */
 511                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSC2BW*", "*" },
 512                 /*quirks*/ADA_Q_4K
 513         },
 514         {
 515                 /*
 516                  * Intel X25-M Series SSDs
 517                  * 4k optimised & trim only works in 4k requests + 4k aligned
 518                  */
 519                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "INTEL SSDSA2M*", "*" },
 520                 /*quirks*/ADA_Q_4K
 521         },
 522         {
 523                 /*
 524                  * Kingston E100 Series SSDs
 525                  * 4k optimised & trim only works in 4k requests + 4k aligned
 526                  */
 527                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SE100S3*", "*" },
 528                 /*quirks*/ADA_Q_4K
 529         },
 530         {
 531                 /*
 532                  * Kingston HyperX 3k SSDs
 533                  * 4k optimised & trim only works in 4k requests + 4k aligned
 534                  */
 535                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "KINGSTON SH103S3*", "*" },
 536                 /*quirks*/ADA_Q_4K
 537         },
 538         {
 539                 /*
 540                  * Marvell SSDs (entry taken from OpenSolaris)
 541                  * 4k optimised & trim only works in 4k requests + 4k aligned
 542                  */
 543                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "MARVELL SD88SA02*", "*" },
 544                 /*quirks*/ADA_Q_4K
 545         },
 546         {
 547                 /*
 548                  * Micron M500 SSDs firmware MU07
 549                  * NCQ Trim works?
 550                  */
 551                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "MU07" },
 552                 /*quirks*/0
 553         },
 554         {
 555                 /*
 556                  * Micron M500 SSDs all other firmware
 557                  * NCQ Trim doesn't work
 558                  */
 559                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M500*", "*" },
 560                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 561         },
 562         {
 563                 /*
 564                  * Micron M5[15]0 SSDs
 565                  * NCQ Trim doesn't work, but only MU01 firmware
 566                  */
 567                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Micron M5[15]0*", "MU01" },
 568                 /*quirks*/ADA_Q_NCQ_TRIM_BROKEN
 569         },
 570         {
 571                 /*
 572                  * OCZ Agility 2 SSDs
 573                  * 4k optimised & trim only works in 4k requests + 4k aligned
 574                  */
 575                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY2*", "*" },
 576                 /*quirks*/ADA_Q_4K
 577         },
 578         {
 579                 /*
 580                  * OCZ Agility 3 SSDs
 581                  * 4k optimised & trim only works in 4k requests + 4k aligned
 582                  */
 583                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-AGILITY3*", "*" },
 584                 /*quirks*/ADA_Q_4K
 585         },
 586         {
 587                 /*
 588                  * OCZ Deneva R Series SSDs
 589                  * 4k optimised & trim only works in 4k requests + 4k aligned
 590                  */
 591                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "DENRSTE251M45*", "*" },
 592                 /*quirks*/ADA_Q_4K
 593         },
 594         {
 595                 /*
 596                  * OCZ Vertex 2 SSDs (inc pro series)
 597                  * 4k optimised & trim only works in 4k requests + 4k aligned
 598                  */
 599                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ?VERTEX2*", "*" },
 600                 /*quirks*/ADA_Q_4K
 601         },
 602         {
 603                 /*
 604                  * OCZ Vertex 3 SSDs
 605                  * 4k optimised & trim only works in 4k requests + 4k aligned
 606                  */
 607                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX3*", "*" },
 608                 /*quirks*/ADA_Q_4K
 609         },
 610         {
 611                 /*
 612                  * OCZ Vertex 4 SSDs
 613                  * 4k optimised & trim only works in 4k requests + 4k aligned
 614                  */
 615                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "OCZ-VERTEX4*", "*" },
 616                 /*quirks*/ADA_Q_4K
 617         },
 618         {
 619                 /*
 620                  * Samsung 830 Series SSDs
 621                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 622                  */
 623                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD 830 Series*", "*" },
 624                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 625         },
 626         {
 627                 /*
 628                  * Samsung 840 SSDs
 629                  * 4k optimised, NCQ TRIM Broken (normal TRIM is fine)
 630                  */
 631                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 840*", "*" },
 632                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 633         },
 634         {
 635                 /*
 636                  * Samsung 850 SSDs
 637                  * 4k optimised, NCQ TRIM broken (normal TRIM fine)
 638                  */
 639                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "Samsung SSD 850*", "*" },
 640                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 641         },
 642         {
 643                 /*
 644                  * Samsung SM863 Series SSDs (MZ7KM*)
 645                  * 4k optimised, NCQ believed to be working
 646                  */
 647                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7KM*", "*" },
 648                 /*quirks*/ADA_Q_4K
 649         },
 650         {
 651                 /*
 652                  * Samsung 843T Series SSDs (MZ7WD*)
 653                  * Samsung PM851 Series SSDs (MZ7TE*)
 654                  * Samsung PM853T Series SSDs (MZ7GE*)
 655                  * 4k optimised, NCQ believed to be broken since these are
 656                  * appear to be built with the same controllers as the 840/850.
 657                  */
 658                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG MZ7*", "*" },
 659                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 660         },
 661         {
 662                 /*
 663                  * Samsung PM851 Series SSDs Dell OEM
 664                  * device model          "SAMSUNG SSD PM851 mSATA 256GB"
 665                  * 4k optimised, NCQ broken
 666                  */
 667                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG SSD PM851*", "*" },
 668                 /*quirks*/ADA_Q_4K | ADA_Q_NCQ_TRIM_BROKEN
 669         },
 670         {
 671                 /*
 672                  * SuperTalent TeraDrive CT SSDs
 673                  * 4k optimised & trim only works in 4k requests + 4k aligned
 674                  */
 675                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "FTM??CT25H*", "*" },
 676                 /*quirks*/ADA_Q_4K
 677         },
 678         {
 679                 /*
 680                  * XceedIOPS SATA SSDs
 681                  * 4k optimised
 682                  */
 683                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SG9XCS2D*", "*" },
 684                 /*quirks*/ADA_Q_4K
 685         },
 686         {
 687                 /*
 688                  * Samsung drive that doesn't support READ LOG EXT or
 689                  * READ LOG DMA EXT, despite reporting that it does in
 690                  * ATA identify data:
 691                  * SAMSUNG HD200HJ KF100-06
 692                  */
 693                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD200*", "*" },
 694                 /*quirks*/ADA_Q_LOG_BROKEN
 695         },
 696         {
 697                 /*
 698                  * Samsung drive that doesn't support READ LOG EXT or
 699                  * READ LOG DMA EXT, despite reporting that it does in
 700                  * ATA identify data:
 701                  * SAMSUNG HD501LJ CR100-10
 702                  */
 703                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "SAMSUNG HD501*", "*" },
 704                 /*quirks*/ADA_Q_LOG_BROKEN
 705         },
 706         {
 707                 /*
 708                  * Seagate Lamarr 8TB Shingled Magnetic Recording (SMR)
 709                  * Drive Managed SATA hard drive.  This drive doesn't report
 710                  * in firmware that it is a drive managed SMR drive.
 711                  */
 712                 { T_DIRECT, SIP_MEDIA_FIXED, "*", "ST8000AS0002*", "*" },
 713                 /*quirks*/ADA_Q_SMR_DM
 714         },
 715         {
 716                 /* Default */
 717                 {
 718                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 719                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 720                 },
 721                 /*quirks*/0
 722         },
 723 };
 724
 725 static  disk_strategy_t adastrategy;
 726 static  dumper_t        adadump;
 727 static  periph_init_t   adainit;
 728 static  void            adadiskgonecb(struct disk *dp);
 729 static  periph_oninv_t  adaoninvalidate;
 730 static  periph_dtor_t   adacleanup;
 731 static  void            adaasync(void *callback_arg, u_int32_t code,
 732                                 struct cam_path *path, void *arg);
 733 static  int             adazonemodesysctl(SYSCTL_HANDLER_ARGS);
 734 static  int             adazonesupsysctl(SYSCTL_HANDLER_ARGS);
 735 static  void            adasysctlinit(void *context, int pending);
 736 static  int             adagetattr(struct bio *bp);
 737 static  void            adasetflags(struct ada_softc *softc,
 738                                     struct ccb_getdev *cgd);
 739 static  periph_ctor_t   adaregister;
 740 static  void            ada_dsmtrim(struct ada_softc *softc, struct bio *bp,
 741                                     struct ccb_ataio *ataio);
 742 static  void            ada_cfaerase(struct ada_softc *softc, struct bio *bp,
 743                                      struct ccb_ataio *ataio);
 744 static  int             ada_zone_bio_to_ata(int disk_zone_cmd);
 745 static  int             ada_zone_cmd(struct cam_periph *periph, union ccb *ccb,
 746                                      struct bio *bp, int *queue_ccb);
 747 static  periph_start_t  adastart;
 748 static  void            adaprobedone(struct cam_periph *periph, union ccb *ccb);
 749 static  void            adazonedone(struct cam_periph *periph, union ccb *ccb);
 750 static  void            adadone(struct cam_periph *periph,
 751                                union ccb *done_ccb);
 752 static  int             adaerror(union ccb *ccb, u_int32_t cam_flags,
 753                                 u_int32_t sense_flags);
 754 static void             adagetparams(struct cam_periph *periph,
 755                                 struct ccb_getdev *cgd);
 756 static timeout_t        adasendorderedtag;
 757 static void             adashutdown(void *arg, int howto);
 758 static void             adasuspend(void *arg);
 759 static void             adaresume(void *arg);
 760
 761 #ifndef ADA_DEFAULT_LEGACY_ALIASES
 762 #define ADA_DEFAULT_LEGACY_ALIASES      1
 763 #endif
 764
 765 #ifndef ADA_DEFAULT_TIMEOUT
 766 #define ADA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 767 #endif
 768
 769 #ifndef ADA_DEFAULT_RETRY
 770 #define ADA_DEFAULT_RETRY       4
 771 #endif
 772
 773 #ifndef ADA_DEFAULT_SEND_ORDERED
 774 #define ADA_DEFAULT_SEND_ORDERED        1
 775 #endif
 776
 777 #ifndef ADA_DEFAULT_SPINDOWN_SHUTDOWN
 778 #define ADA_DEFAULT_SPINDOWN_SHUTDOWN   1
 779 #endif
 780
 781 #ifndef ADA_DEFAULT_SPINDOWN_SUSPEND
 782 #define ADA_DEFAULT_SPINDOWN_SUSPEND    1
 783 #endif
 784
 785 #ifndef ADA_DEFAULT_READ_AHEAD
 786 #define ADA_DEFAULT_READ_AHEAD  1
 787 #endif
 788
 789 #ifndef ADA_DEFAULT_WRITE_CACHE
 790 #define ADA_DEFAULT_WRITE_CACHE 1
 791 #endif
 792
 793 #define ADA_RA  (softc->read_ahead >= 0 ? \
 794                  softc->read_ahead : ada_read_ahead)
 795 #define ADA_WC  (softc->write_cache >= 0 ? \
 796                  softc->write_cache : ada_write_cache)
 797
 798 /*
 799  * Most platforms map firmware geometry to actual, but some don't.  If
 800  * not overridden, default to nothing.
 801  */
 802 #ifndef ata_disk_firmware_geom_adjust
 803 #define ata_disk_firmware_geom_adjust(disk)
 804 #endif
 805
 806 static int ada_retry_count = ADA_DEFAULT_RETRY;
 807 static int ada_default_timeout = ADA_DEFAULT_TIMEOUT;
 808 static int ada_send_ordered = ADA_DEFAULT_SEND_ORDERED;
 809 static int ada_spindown_shutdown = ADA_DEFAULT_SPINDOWN_SHUTDOWN;
 810 static int ada_spindown_suspend = ADA_DEFAULT_SPINDOWN_SUSPEND;
 811 static int ada_read_ahead = ADA_DEFAULT_READ_AHEAD;
 812 static int ada_write_cache = ADA_DEFAULT_WRITE_CACHE;
 813
 814 static SYSCTL_NODE(_kern_cam, OID_AUTO, ada, CTLFLAG_RD, 0,
 815             "CAM Direct Access Disk driver");
 816 SYSCTL_INT(_kern_cam_ada, OID_AUTO, retry_count, CTLFLAG_RWTUN,
 817            &ada_retry_count, 0, "Normal I/O retry count");
 818 SYSCTL_INT(_kern_cam_ada, OID_AUTO, default_timeout, CTLFLAG_RWTUN,
 819            &ada_default_timeout, 0, "Normal I/O timeout (in seconds)");
 820 SYSCTL_INT(_kern_cam_ada, OID_AUTO, send_ordered, CTLFLAG_RWTUN,
 821            &ada_send_ordered, 0, "Send Ordered Tags");
 822 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_shutdown, CTLFLAG_RWTUN,
 823            &ada_spindown_shutdown, 0, "Spin down upon shutdown");
 824 SYSCTL_INT(_kern_cam_ada, OID_AUTO, spindown_suspend, CTLFLAG_RWTUN,
 825            &ada_spindown_suspend, 0, "Spin down upon suspend");
 826 SYSCTL_INT(_kern_cam_ada, OID_AUTO, read_ahead, CTLFLAG_RWTUN,
 827            &ada_read_ahead, 0, "Enable disk read-ahead");
 828 SYSCTL_INT(_kern_cam_ada, OID_AUTO, write_cache, CTLFLAG_RWTUN,
 829            &ada_write_cache, 0, "Enable disk write cache");
 830
 831 /*
 832  * ADA_ORDEREDTAG_INTERVAL determines how often, relative
 833  * to the default timeout, we check to see whether an ordered
 834  * tagged transaction is appropriate to prevent simple tag
 835  * starvation.  Since we'd like to ensure that there is at least
 836  * 1/2 of the timeout length left for a starved transaction to
 837  * complete after we've sent an ordered tag, we must poll at least
 838  * four times in every timeout period.  This takes care of the worst
 839  * case where a starved transaction starts during an interval that
 840  * meets the requirement "don't send an ordered tag" test so it takes
 841  * us two intervals to determine that a tag must be sent.
 842  */
 843 #ifndef ADA_ORDEREDTAG_INTERVAL
 844 #define ADA_ORDEREDTAG_INTERVAL 4
 845 #endif
 846
 847 static struct periph_driver adadriver =
 848 {
 849         adainit, "ada",
 850         TAILQ_HEAD_INITIALIZER(adadriver.units), /* generation */ 0
 851 };
 852
 853 static int adadeletemethodsysctl(SYSCTL_HANDLER_ARGS);
 854
 855 PERIPHDRIVER_DECLARE(ada, adadriver);
 856
 857 static MALLOC_DEFINE(M_ATADA, "ata_da", "ata_da buffers");
 858
 859 static int
 860 adaopen(struct disk *dp)
 861 {
 862         struct cam_periph *periph;
 863         struct ada_softc *softc;
 864         int error;
 865
 866         periph = (struct cam_periph *)dp->d_drv1;
 867         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 868                 return(ENXIO);
 869         }
 870
 871         cam_periph_lock(periph);
 872         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 873                 cam_periph_unlock(periph);
 874                 cam_periph_release(periph);
 875                 return (error);
 876         }
 877
 878         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 879             ("adaopen\n"));
 880
 881         softc = (struct ada_softc *)periph->softc;
 882         softc->flags |= ADA_FLAG_OPEN;
 883
 884         cam_periph_unhold(periph);
 885         cam_periph_unlock(periph);
 886         return (0);
 887 }
 888
 889 static int
 890 adaclose(struct disk *dp)
 891 {
 892         struct  cam_periph *periph;
 893         struct  ada_softc *softc;
 894         union ccb *ccb;
 895         int error;
 896
 897         periph = (struct cam_periph *)dp->d_drv1;
 898         softc = (struct ada_softc *)periph->softc;
 899         cam_periph_lock(periph);
 900
 901         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 902             ("adaclose\n"));
 903
 904         /* We only sync the cache if the drive is capable of it. */
 905         if ((softc->flags & ADA_FLAG_DIRTY) != 0 &&
 906             (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) != 0 &&
 907             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 908             cam_periph_hold(periph, PRIBIO) == 0) {
 909
 910                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 911                 cam_fill_ataio(&ccb->ataio,
 912                                     1,
 913                                     adadone,
 914                                     CAM_DIR_NONE,
 915                                     0,
 916                                     NULL,
 917                                     0,
 918                                     ada_default_timeout*1000);
 919
 920                 if (softc->flags & ADA_FLAG_CAN_48BIT)
 921                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
 922                 else
 923                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
 924                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
 925                     /*sense_flags*/0, softc->disk->d_devstat);
 926
 927                 if (error != 0)
 928                         xpt_print(periph->path, "Synchronize cache failed\n");
 929                 else
 930                         softc->flags &= ~ADA_FLAG_DIRTY;
 931                 xpt_release_ccb(ccb);
 932                 cam_periph_unhold(periph);
 933         }
 934
 935         softc->flags &= ~ADA_FLAG_OPEN;
 936
 937         while (softc->refcount != 0)
 938                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "adaclose", 1);
 939         cam_periph_unlock(periph);
 940         cam_periph_release(periph);
 941         return (0);
 942 }
 943
 944 static void
 945 adaschedule(struct cam_periph *periph)
 946 {
 947         struct ada_softc *softc = (struct ada_softc *)periph->softc;
 948
 949         if (softc->state != ADA_STATE_NORMAL)
 950                 return;
 951
 952         cam_iosched_schedule(softc->cam_iosched, periph);
 953 }
 954
 955 /*
 956  * Actually translate the requested transfer into one the physical driver
 957  * can understand.  The transfer is described by a buf and will include
 958  * only one physical transfer.
 959  */
 960 static void
 961 adastrategy(struct bio *bp)
 962 {
 963         struct cam_periph *periph;
 964         struct ada_softc *softc;
 965
 966         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
 967         softc = (struct ada_softc *)periph->softc;
 968
 969         cam_periph_lock(periph);
 970
 971         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastrategy(%p)\n", bp));
 972
 973         /*
 974          * If the device has been made invalid, error out
 975          */
 976         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 977                 cam_periph_unlock(periph);
 978                 biofinish(bp, NULL, ENXIO);
 979                 return;
 980         }
 981
 982         /*
 983          * Zone commands must be ordered, because they can depend on the
 984          * effects of previously issued commands, and they may affect
 985          * commands after them.
 986          */
 987         if (bp->bio_cmd == BIO_ZONE)
 988                 bp->bio_flags |= BIO_ORDERED;
 989
 990         /*
 991          * Place it in the queue of disk activities for this disk
 992          */
 993         cam_iosched_queue_work(softc->cam_iosched, bp);
 994
 995         /*
 996          * Schedule ourselves for performing the work.
 997          */
 998         adaschedule(periph);
 999         cam_periph_unlock(periph);
1000
1001         return;
1002 }
1003
1004 static int
1005 adadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
1006 {
1007         struct      cam_periph *periph;
1008         struct      ada_softc *softc;
1009         u_int       secsize;
1010         union       ccb ccb;
1011         struct      disk *dp;
1012         uint64_t    lba;
1013         uint16_t    count;
1014         int         error = 0;
1015
1016         dp = arg;
1017         periph = dp->d_drv1;
1018         softc = (struct ada_softc *)periph->softc;
1019         cam_periph_lock(periph);
1020         secsize = softc->params.secsize;
1021         lba = offset / secsize;
1022         count = length / secsize;
1023
1024         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1025                 cam_periph_unlock(periph);
1026                 return (ENXIO);
1027         }
1028
1029         if (length > 0) {
1030                 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1031                 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
1032                 cam_fill_ataio(&ccb.ataio,
1033                     0,
1034                     adadone,
1035                     CAM_DIR_OUT,
1036                     0,
1037                     (u_int8_t *) virtual,
1038                     length,
1039                     ada_default_timeout*1000);
1040                 if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
1041                     (lba + count >= ATA_MAX_28BIT_LBA ||
1042                     count >= 256)) {
1043                         ata_48bit_cmd(&ccb.ataio, ATA_WRITE_DMA48,
1044                             0, lba, count);
1045                 } else {
1046                         ata_28bit_cmd(&ccb.ataio, ATA_WRITE_DMA,
1047                             0, lba, count);
1048                 }
1049                 xpt_polled_action(&ccb);
1050
1051                 error = cam_periph_error(&ccb,
1052                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1053                 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
1054                         cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
1055                             /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
1056                 if (error != 0)
1057                         printf("Aborting dump due to I/O error.\n");
1058
1059                 cam_periph_unlock(periph);
1060                 return (error);
1061         }
1062
1063         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) {
1064                 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1065
1066                 /*
1067                  * Tell the drive to flush its internal cache. if we
1068                  * can't flush in 5s we have big problems. No need to
1069                  * wait the default 60s to detect problems.
1070                  */
1071                 ccb.ccb_h.ccb_state = ADA_CCB_DUMP;
1072                 cam_fill_ataio(&ccb.ataio,
1073                                     0,
1074                                     adadone,
1075                                     CAM_DIR_NONE,
1076                                     0,
1077                                     NULL,
1078                                     0,
1079                                     5*1000);
1080
1081                 if (softc->flags & ADA_FLAG_CAN_48BIT)
1082                         ata_48bit_cmd(&ccb.ataio, ATA_FLUSHCACHE48, 0, 0, 0);
1083                 else
1084                         ata_28bit_cmd(&ccb.ataio, ATA_FLUSHCACHE, 0, 0, 0);
1085                 xpt_polled_action(&ccb);
1086
1087                 error = cam_periph_error(&ccb,
1088                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
1089                 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
1090                         cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
1091                             /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
1092                 if (error != 0)
1093                         xpt_print(periph->path, "Synchronize cache failed\n");
1094         }
1095         cam_periph_unlock(periph);
1096         return (error);
1097 }
1098
1099 static void
1100 adainit(void)
1101 {
1102         cam_status status;
1103
1104         /*
1105          * Install a global async callback.  This callback will
1106          * receive async callbacks like "new device found".
1107          */
1108         status = xpt_register_async(AC_FOUND_DEVICE, adaasync, NULL, NULL);
1109
1110         if (status != CAM_REQ_CMP) {
1111                 printf("ada: Failed to attach master async callback "
1112                        "due to status 0x%x!\n", status);
1113         } else if (ada_send_ordered) {
1114
1115                 /* Register our event handlers */
1116                 if ((EVENTHANDLER_REGISTER(power_suspend, adasuspend,
1117                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1118                     printf("adainit: power event registration failed!\n");
1119                 if ((EVENTHANDLER_REGISTER(power_resume, adaresume,
1120                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
1121                     printf("adainit: power event registration failed!\n");
1122                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, adashutdown,
1123                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
1124                     printf("adainit: shutdown event registration failed!\n");
1125         }
1126 }
1127
1128 /*
1129  * Callback from GEOM, called when it has finished cleaning up its
1130  * resources.
1131  */
1132 static void
1133 adadiskgonecb(struct disk *dp)
1134 {
1135         struct cam_periph *periph;
1136
1137         periph = (struct cam_periph *)dp->d_drv1;
1138
1139         cam_periph_release(periph);
1140 }
1141
1142 static void
1143 adaoninvalidate(struct cam_periph *periph)
1144 {
1145         struct ada_softc *softc;
1146
1147         softc = (struct ada_softc *)periph->softc;
1148
1149         /*
1150          * De-register any async callbacks.
1151          */
1152         xpt_register_async(0, adaasync, periph, periph->path);
1153 #ifdef CAM_IO_STATS
1154         softc->invalidations++;
1155 #endif
1156
1157         /*
1158          * Return all queued I/O with ENXIO.
1159          * XXX Handle any transactions queued to the card
1160          *     with XPT_ABORT_CCB.
1161          */
1162         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
1163
1164         disk_gone(softc->disk);
1165 }
1166
1167 static void
1168 adacleanup(struct cam_periph *periph)
1169 {
1170         struct ada_softc *softc;
1171
1172         softc = (struct ada_softc *)periph->softc;
1173
1174         cam_periph_unlock(periph);
1175
1176         cam_iosched_fini(softc->cam_iosched);
1177
1178         /*
1179          * If we can't free the sysctl tree, oh well...
1180          */
1181         if ((softc->flags & ADA_FLAG_SCTX_INIT) != 0) {
1182 #ifdef CAM_IO_STATS
1183                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
1184                         xpt_print(periph->path,
1185                             "can't remove sysctl stats context\n");
1186 #endif
1187                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
1188                         xpt_print(periph->path,
1189                             "can't remove sysctl context\n");
1190         }
1191
1192         disk_destroy(softc->disk);
1193         callout_drain(&softc->sendordered_c);
1194         free(softc, M_DEVBUF);
1195         cam_periph_lock(periph);
1196 }
1197
1198 static void
1199 adasetdeletemethod(struct ada_softc *softc)
1200 {
1201
1202         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1203                 softc->delete_method = ADA_DELETE_NCQ_DSM_TRIM;
1204         else if (softc->flags & ADA_FLAG_CAN_TRIM)
1205                 softc->delete_method = ADA_DELETE_DSM_TRIM;
1206         else if ((softc->flags & ADA_FLAG_CAN_CFA) && !(softc->flags & ADA_FLAG_CAN_48BIT))
1207                 softc->delete_method = ADA_DELETE_CFA_ERASE;
1208         else
1209                 softc->delete_method = ADA_DELETE_NONE;
1210 }
1211
1212 static void
1213 adaasync(void *callback_arg, u_int32_t code,
1214         struct cam_path *path, void *arg)
1215 {
1216         struct ccb_getdev cgd;
1217         struct cam_periph *periph;
1218         struct ada_softc *softc;
1219
1220         periph = (struct cam_periph *)callback_arg;
1221         switch (code) {
1222         case AC_FOUND_DEVICE:
1223         {
1224                 struct ccb_getdev *cgd;
1225                 cam_status status;
1226
1227                 cgd = (struct ccb_getdev *)arg;
1228                 if (cgd == NULL)
1229                         break;
1230
1231                 if (cgd->protocol != PROTO_ATA)
1232                         break;
1233
1234                 /*
1235                  * Allocate a peripheral instance for
1236                  * this device and start the probe
1237                  * process.
1238                  */
1239                 status = cam_periph_alloc(adaregister, adaoninvalidate,
1240                                           adacleanup, adastart,
1241                                           "ada", CAM_PERIPH_BIO,
1242                                           path, adaasync,
1243                                           AC_FOUND_DEVICE, cgd);
1244
1245                 if (status != CAM_REQ_CMP
1246                  && status != CAM_REQ_INPROG)
1247                         printf("adaasync: Unable to attach to new device "
1248                                 "due to status 0x%x\n", status);
1249                 break;
1250         }
1251         case AC_GETDEV_CHANGED:
1252         {
1253                 softc = (struct ada_softc *)periph->softc;
1254                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1255                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1256                 xpt_action((union ccb *)&cgd);
1257
1258                 /*
1259                  * Set/clear support flags based on the new Identify data.
1260                  */
1261                 adasetflags(softc, &cgd);
1262
1263                 cam_periph_async(periph, code, path, arg);
1264                 break;
1265         }
1266         case AC_ADVINFO_CHANGED:
1267         {
1268                 uintptr_t buftype;
1269
1270                 buftype = (uintptr_t)arg;
1271                 if (buftype == CDAI_TYPE_PHYS_PATH) {
1272                         struct ada_softc *softc;
1273
1274                         softc = periph->softc;
1275                         disk_attr_changed(softc->disk, "GEOM::physpath",
1276                                           M_NOWAIT);
1277                 }
1278                 break;
1279         }
1280         case AC_SENT_BDR:
1281         case AC_BUS_RESET:
1282         {
1283                 softc = (struct ada_softc *)periph->softc;
1284                 cam_periph_async(periph, code, path, arg);
1285                 if (softc->state != ADA_STATE_NORMAL)
1286                         break;
1287                 xpt_setup_ccb(&cgd.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
1288                 cgd.ccb_h.func_code = XPT_GDEV_TYPE;
1289                 xpt_action((union ccb *)&cgd);
1290                 if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1291                         softc->state = ADA_STATE_RAHEAD;
1292                 else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD)
1293                         softc->state = ADA_STATE_WCACHE;
1294                 else if ((softc->flags & ADA_FLAG_CAN_LOG)
1295                       && (softc->zone_mode != ADA_ZONE_NONE))
1296                         softc->state = ADA_STATE_LOGDIR;
1297                 else
1298                     break;
1299                 if (cam_periph_acquire(periph) != CAM_REQ_CMP)
1300                         softc->state = ADA_STATE_NORMAL;
1301                 else
1302                         xpt_schedule(periph, CAM_PRIORITY_DEV);
1303         }
1304         default:
1305                 cam_periph_async(periph, code, path, arg);
1306                 break;
1307         }
1308 }
1309
1310 static int
1311 adazonemodesysctl(SYSCTL_HANDLER_ARGS)
1312 {
1313         char tmpbuf[40];
1314         struct ada_softc *softc;
1315         int error;
1316
1317         softc = (struct ada_softc *)arg1;
1318
1319         switch (softc->zone_mode) {
1320         case ADA_ZONE_DRIVE_MANAGED:
1321                 snprintf(tmpbuf, sizeof(tmpbuf), "Drive Managed");
1322                 break;
1323         case ADA_ZONE_HOST_AWARE:
1324                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Aware");
1325                 break;
1326         case ADA_ZONE_HOST_MANAGED:
1327                 snprintf(tmpbuf, sizeof(tmpbuf), "Host Managed");
1328                 break;
1329         case ADA_ZONE_NONE:
1330         default:
1331                 snprintf(tmpbuf, sizeof(tmpbuf), "Not Zoned");
1332                 break;
1333         }
1334
1335         error = sysctl_handle_string(oidp, tmpbuf, sizeof(tmpbuf), req);
1336
1337         return (error);
1338 }
1339
1340 static int
1341 adazonesupsysctl(SYSCTL_HANDLER_ARGS)
1342 {
1343         char tmpbuf[180];
1344         struct ada_softc *softc;
1345         struct sbuf sb;
1346         int error, first;
1347         unsigned int i;
1348
1349         softc = (struct ada_softc *)arg1;
1350
1351         error = 0;
1352         first = 1;
1353         sbuf_new(&sb, tmpbuf, sizeof(tmpbuf), 0);
1354
1355         for (i = 0; i < sizeof(ada_zone_desc_table) /
1356              sizeof(ada_zone_desc_table[0]); i++) {
1357                 if (softc->zone_flags & ada_zone_desc_table[i].value) {
1358                         if (first == 0)
1359                                 sbuf_printf(&sb, ", ");
1360                         else
1361                                 first = 0;
1362                         sbuf_cat(&sb, ada_zone_desc_table[i].desc);
1363                 }
1364         }
1365
1366         if (first == 1)
1367                 sbuf_printf(&sb, "None");
1368
1369         sbuf_finish(&sb);
1370
1371         error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
1372
1373         return (error);
1374 }
1375
1376
1377 static void
1378 adasysctlinit(void *context, int pending)
1379 {
1380         struct cam_periph *periph;
1381         struct ada_softc *softc;
1382         char tmpstr[80], tmpstr2[80];
1383
1384         periph = (struct cam_periph *)context;
1385
1386         /* periph was held for us when this task was enqueued */
1387         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
1388                 cam_periph_release(periph);
1389                 return;
1390         }
1391
1392         softc = (struct ada_softc *)periph->softc;
1393         snprintf(tmpstr, sizeof(tmpstr), "CAM ADA unit %d",periph->unit_number);
1394         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
1395
1396         sysctl_ctx_init(&softc->sysctl_ctx);
1397         softc->flags |= ADA_FLAG_SCTX_INIT;
1398         softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
1399                 SYSCTL_STATIC_CHILDREN(_kern_cam_ada), OID_AUTO, tmpstr2,
1400                 CTLFLAG_RD, 0, tmpstr);
1401         if (softc->sysctl_tree == NULL) {
1402                 printf("adasysctlinit: unable to allocate sysctl tree\n");
1403                 cam_periph_release(periph);
1404                 return;
1405         }
1406
1407         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1408                 OID_AUTO, "delete_method", CTLTYPE_STRING | CTLFLAG_RW,
1409                 softc, 0, adadeletemethodsysctl, "A",
1410                 "BIO_DELETE execution method");
1411         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1412                 OID_AUTO, "read_ahead", CTLFLAG_RW | CTLFLAG_MPSAFE,
1413                 &softc->read_ahead, 0, "Enable disk read ahead.");
1414         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1415                 OID_AUTO, "write_cache", CTLFLAG_RW | CTLFLAG_MPSAFE,
1416                 &softc->write_cache, 0, "Enable disk write cache.");
1417         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1418                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
1419                 &softc->unmappedio, 0, "Unmapped I/O leaf");
1420         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1421                 OID_AUTO, "rotating", CTLFLAG_RD | CTLFLAG_MPSAFE,
1422                 &softc->rotating, 0, "Rotating media");
1423         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1424                 OID_AUTO, "zone_mode", CTLTYPE_STRING | CTLFLAG_RD,
1425                 softc, 0, adazonemodesysctl, "A",
1426                 "Zone Mode");
1427         SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1428                 OID_AUTO, "zone_support", CTLTYPE_STRING | CTLFLAG_RD,
1429                 softc, 0, adazonesupsysctl, "A",
1430                 "Zone Support");
1431         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1432                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1433                 "optimal_seq_zones", CTLFLAG_RD, &softc->optimal_seq_zones,
1434                 "Optimal Number of Open Sequential Write Preferred Zones");
1435         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1436                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1437                 "optimal_nonseq_zones", CTLFLAG_RD,
1438                 &softc->optimal_nonseq_zones,
1439                 "Optimal Number of Non-Sequentially Written Sequential Write "
1440                 "Preferred Zones");
1441         SYSCTL_ADD_UQUAD(&softc->sysctl_ctx,
1442                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO,
1443                 "max_seq_zones", CTLFLAG_RD, &softc->max_seq_zones,
1444                 "Maximum Number of Open Sequential Write Required Zones");
1445
1446 #ifdef ADA_TEST_FAILURE
1447         /*
1448          * Add a 'door bell' sysctl which allows one to set it from userland
1449          * and cause something bad to happen.  For the moment, we only allow
1450          * whacking the next read or write.
1451          */
1452         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1453                 OID_AUTO, "force_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1454                 &softc->force_read_error, 0,
1455                 "Force a read error for the next N reads.");
1456         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1457                 OID_AUTO, "force_write_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1458                 &softc->force_write_error, 0,
1459                 "Force a write error for the next N writes.");
1460         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
1461                 OID_AUTO, "periodic_read_error", CTLFLAG_RW | CTLFLAG_MPSAFE,
1462                 &softc->periodic_read_error, 0,
1463                 "Force a read error every N reads (don't set too low).");
1464 #endif
1465
1466 #ifdef CAM_IO_STATS
1467         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
1468                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
1469                 CTLFLAG_RD, 0, "Statistics");
1470         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1471                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1472                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
1473                 &softc->timeouts, 0,
1474                 "Device timeouts reported by the SIM");
1475         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1476                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1477                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
1478                 &softc->errors, 0,
1479                 "Transport errors reported by the SIM.");
1480         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
1481                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
1482                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
1483                 &softc->invalidations, 0,
1484                 "Device pack invalidations.");
1485 #endif
1486
1487         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
1488             softc->sysctl_tree);
1489
1490         cam_periph_release(periph);
1491 }
1492
1493 static int
1494 adagetattr(struct bio *bp)
1495 {
1496         int ret;
1497         struct cam_periph *periph;
1498
1499         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
1500         cam_periph_lock(periph);
1501         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
1502             periph->path);
1503         cam_periph_unlock(periph);
1504         if (ret == 0)
1505                 bp->bio_completed = bp->bio_length;
1506         return ret;
1507 }
1508
1509 static int
1510 adadeletemethodsysctl(SYSCTL_HANDLER_ARGS)
1511 {
1512         char buf[16];
1513         const char *p;
1514         struct ada_softc *softc;
1515         int i, error, value, methods;
1516
1517         softc = (struct ada_softc *)arg1;
1518
1519         value = softc->delete_method;
1520         if (value < 0 || value > ADA_DELETE_MAX)
1521                 p = "UNKNOWN";
1522         else
1523                 p = ada_delete_method_names[value];
1524         strncpy(buf, p, sizeof(buf));
1525         error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
1526         if (error != 0 || req->newptr == NULL)
1527                 return (error);
1528         methods = 1 << ADA_DELETE_DISABLE;
1529         if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1530             !(softc->flags & ADA_FLAG_CAN_48BIT))
1531                 methods |= 1 << ADA_DELETE_CFA_ERASE;
1532         if (softc->flags & ADA_FLAG_CAN_TRIM)
1533                 methods |= 1 << ADA_DELETE_DSM_TRIM;
1534         if (softc->flags & ADA_FLAG_CAN_NCQ_TRIM)
1535                 methods |= 1 << ADA_DELETE_NCQ_DSM_TRIM;
1536         for (i = 0; i <= ADA_DELETE_MAX; i++) {
1537                 if (!(methods & (1 << i)) ||
1538                     strcmp(buf, ada_delete_method_names[i]) != 0)
1539                         continue;
1540                 softc->delete_method = i;
1541                 return (0);
1542         }
1543         return (EINVAL);
1544 }
1545
1546 static void
1547 adasetflags(struct ada_softc *softc, struct ccb_getdev *cgd)
1548 {
1549         if ((cgd->ident_data.capabilities1 & ATA_SUPPORT_DMA) &&
1550             (cgd->inq_flags & SID_DMA))
1551                 softc->flags |= ADA_FLAG_CAN_DMA;
1552         else
1553                 softc->flags &= ~ADA_FLAG_CAN_DMA;
1554
1555         if (cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) {
1556                 softc->flags |= ADA_FLAG_CAN_48BIT;
1557                 if (cgd->inq_flags & SID_DMA48)
1558                         softc->flags |= ADA_FLAG_CAN_DMA48;
1559                 else
1560                         softc->flags &= ~ADA_FLAG_CAN_DMA48;
1561         } else
1562                 softc->flags &= ~(ADA_FLAG_CAN_48BIT | ADA_FLAG_CAN_DMA48);
1563
1564         if (cgd->ident_data.support.command2 & ATA_SUPPORT_FLUSHCACHE)
1565                 softc->flags |= ADA_FLAG_CAN_FLUSHCACHE;
1566         else
1567                 softc->flags &= ~ADA_FLAG_CAN_FLUSHCACHE;
1568
1569         if (cgd->ident_data.support.command1 & ATA_SUPPORT_POWERMGT)
1570                 softc->flags |= ADA_FLAG_CAN_POWERMGT;
1571         else
1572                 softc->flags &= ~ADA_FLAG_CAN_POWERMGT;
1573
1574         if ((cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ) &&
1575             (cgd->inq_flags & SID_DMA) && (cgd->inq_flags & SID_CmdQue))
1576                 softc->flags |= ADA_FLAG_CAN_NCQ;
1577         else
1578                 softc->flags &= ~ADA_FLAG_CAN_NCQ;
1579
1580         if ((cgd->ident_data.support_dsm & ATA_SUPPORT_DSM_TRIM) &&
1581             (cgd->inq_flags & SID_DMA)) {
1582                 softc->flags |= ADA_FLAG_CAN_TRIM;
1583                 softc->trim_max_ranges = TRIM_MAX_RANGES;
1584                 if (cgd->ident_data.max_dsm_blocks != 0) {
1585                         softc->trim_max_ranges =
1586                             min(cgd->ident_data.max_dsm_blocks *
1587                                 ATA_DSM_BLK_RANGES, softc->trim_max_ranges);
1588                 }
1589                 /*
1590                  * If we can do RCVSND_FPDMA_QUEUED commands, we may be able
1591                  * to do NCQ trims, if we support trims at all. We also need
1592                  * support from the SIM to do things properly. Perhaps we
1593                  * should look at log 13 dword 0 bit 0 and dword 1 bit 0 are
1594                  * set too...
1595                  */
1596                 if ((softc->quirks & ADA_Q_NCQ_TRIM_BROKEN) == 0 &&
1597                     (softc->flags & ADA_FLAG_PIM_ATA_EXT) != 0 &&
1598                     (cgd->ident_data.satacapabilities2 &
1599                      ATA_SUPPORT_RCVSND_FPDMA_QUEUED) != 0 &&
1600                     (softc->flags & ADA_FLAG_CAN_TRIM) != 0)
1601                         softc->flags |= ADA_FLAG_CAN_NCQ_TRIM;
1602                 else
1603                         softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
1604         } else
1605                 softc->flags &= ~(ADA_FLAG_CAN_TRIM | ADA_FLAG_CAN_NCQ_TRIM);
1606
1607         if (cgd->ident_data.support.command2 & ATA_SUPPORT_CFA)
1608                 softc->flags |= ADA_FLAG_CAN_CFA;
1609         else
1610                 softc->flags &= ~ADA_FLAG_CAN_CFA;
1611
1612         /*
1613          * Now that we've set the appropriate flags, setup the delete
1614          * method.
1615          */
1616         adasetdeletemethod(softc);
1617
1618         if ((cgd->ident_data.support.extension & ATA_SUPPORT_GENLOG)
1619          && ((softc->quirks & ADA_Q_LOG_BROKEN) == 0))
1620                 softc->flags |= ADA_FLAG_CAN_LOG;
1621         else
1622                 softc->flags &= ~ADA_FLAG_CAN_LOG;
1623
1624         if ((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1625              ATA_SUPPORT_ZONE_HOST_AWARE)
1626                 softc->zone_mode = ADA_ZONE_HOST_AWARE;
1627         else if (((cgd->ident_data.support3 & ATA_SUPPORT_ZONE_MASK) ==
1628                    ATA_SUPPORT_ZONE_DEV_MANAGED)
1629               || (softc->quirks & ADA_Q_SMR_DM))
1630                 softc->zone_mode = ADA_ZONE_DRIVE_MANAGED;
1631         else
1632                 softc->zone_mode = ADA_ZONE_NONE;
1633
1634         if (cgd->ident_data.support.command1 & ATA_SUPPORT_LOOKAHEAD)
1635                 softc->flags |= ADA_FLAG_CAN_RAHEAD;
1636         else
1637                 softc->flags &= ~ADA_FLAG_CAN_RAHEAD;
1638
1639         if (cgd->ident_data.support.command1 & ATA_SUPPORT_WRITECACHE)
1640                 softc->flags |= ADA_FLAG_CAN_WCACHE;
1641         else
1642                 softc->flags &= ~ADA_FLAG_CAN_WCACHE;
1643 }
1644
1645 static cam_status
1646 adaregister(struct cam_periph *periph, void *arg)
1647 {
1648         struct ada_softc *softc;
1649         struct ccb_pathinq cpi;
1650         struct ccb_getdev *cgd;
1651         char   announce_buf[80];
1652         struct disk_params *dp;
1653         caddr_t match;
1654         u_int maxio;
1655         int quirks;
1656
1657         cgd = (struct ccb_getdev *)arg;
1658         if (cgd == NULL) {
1659                 printf("adaregister: no getdev CCB, can't register device\n");
1660                 return(CAM_REQ_CMP_ERR);
1661         }
1662
1663         softc = (struct ada_softc *)malloc(sizeof(*softc), M_DEVBUF,
1664             M_NOWAIT|M_ZERO);
1665
1666         if (softc == NULL) {
1667                 printf("adaregister: Unable to probe new device. "
1668                     "Unable to allocate softc\n");
1669                 return(CAM_REQ_CMP_ERR);
1670         }
1671
1672         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
1673                 printf("adaregister: Unable to probe new device. "
1674                        "Unable to allocate iosched memory\n");
1675                 free(softc, M_DEVBUF);
1676                 return(CAM_REQ_CMP_ERR);
1677         }
1678
1679         periph->softc = softc;
1680
1681         /*
1682          * See if this device has any quirks.
1683          */
1684         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
1685                                (caddr_t)ada_quirk_table,
1686                                nitems(ada_quirk_table),
1687                                sizeof(*ada_quirk_table), ata_identify_match);
1688         if (match != NULL)
1689                 softc->quirks = ((struct ada_quirk_entry *)match)->quirks;
1690         else
1691                 softc->quirks = ADA_Q_NONE;
1692
1693         bzero(&cpi, sizeof(cpi));
1694         xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NONE);
1695         cpi.ccb_h.func_code = XPT_PATH_INQ;
1696         xpt_action((union ccb *)&cpi);
1697
1698         TASK_INIT(&softc->sysctl_task, 0, adasysctlinit, periph);
1699
1700         /*
1701          * Register this media as a disk
1702          */
1703         (void)cam_periph_hold(periph, PRIBIO);
1704         cam_periph_unlock(periph);
1705         snprintf(announce_buf, sizeof(announce_buf),
1706             "kern.cam.ada.%d.quirks", periph->unit_number);
1707         quirks = softc->quirks;
1708         TUNABLE_INT_FETCH(announce_buf, &quirks);
1709         softc->quirks = quirks;
1710         softc->read_ahead = -1;
1711         snprintf(announce_buf, sizeof(announce_buf),
1712             "kern.cam.ada.%d.read_ahead", periph->unit_number);
1713         TUNABLE_INT_FETCH(announce_buf, &softc->read_ahead);
1714         softc->write_cache = -1;
1715         snprintf(announce_buf, sizeof(announce_buf),
1716             "kern.cam.ada.%d.write_cache", periph->unit_number);
1717         TUNABLE_INT_FETCH(announce_buf, &softc->write_cache);
1718
1719         /*
1720          * Set support flags based on the Identify data and quirks.
1721          */
1722         adasetflags(softc, cgd);
1723
1724         /* Disable queue sorting for non-rotational media by default. */
1725         if (cgd->ident_data.media_rotation_rate == ATA_RATE_NON_ROTATING) {
1726                 softc->rotating = 0;
1727         } else {
1728                 softc->rotating = 1;
1729         }
1730         cam_iosched_set_sort_queue(softc->cam_iosched,  softc->rotating ? -1 : 0);
1731         adagetparams(periph, cgd);
1732         softc->disk = disk_alloc();
1733         softc->disk->d_rotation_rate = cgd->ident_data.media_rotation_rate;
1734         softc->disk->d_devstat = devstat_new_entry(periph->periph_name,
1735                           periph->unit_number, softc->params.secsize,
1736                           DEVSTAT_ALL_SUPPORTED,
1737                           DEVSTAT_TYPE_DIRECT |
1738                           XPORT_DEVSTAT_TYPE(cpi.transport),
1739                           DEVSTAT_PRIORITY_DISK);
1740         softc->disk->d_open = adaopen;
1741         softc->disk->d_close = adaclose;
1742         softc->disk->d_strategy = adastrategy;
1743         softc->disk->d_getattr = adagetattr;
1744         softc->disk->d_dump = adadump;
1745         softc->disk->d_gone = adadiskgonecb;
1746         softc->disk->d_name = "ada";
1747         softc->disk->d_drv1 = periph;
1748         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
1749         if (maxio == 0)
1750                 maxio = DFLTPHYS;       /* traditional default */
1751         else if (maxio > MAXPHYS)
1752                 maxio = MAXPHYS;        /* for safety */
1753         if (softc->flags & ADA_FLAG_CAN_48BIT)
1754                 maxio = min(maxio, 65536 * softc->params.secsize);
1755         else                                    /* 28bit ATA command limit */
1756                 maxio = min(maxio, 256 * softc->params.secsize);
1757         softc->disk->d_maxsize = maxio;
1758         softc->disk->d_unit = periph->unit_number;
1759         softc->disk->d_flags = DISKFLAG_DIRECT_COMPLETION | DISKFLAG_CANZONE;
1760         if (softc->flags & ADA_FLAG_CAN_FLUSHCACHE)
1761                 softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
1762         if (softc->flags & ADA_FLAG_CAN_TRIM) {
1763                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1764                 softc->disk->d_delmaxsize = softc->params.secsize *
1765                                             ATA_DSM_RANGE_MAX *
1766                                             softc->trim_max_ranges;
1767         } else if ((softc->flags & ADA_FLAG_CAN_CFA) &&
1768             !(softc->flags & ADA_FLAG_CAN_48BIT)) {
1769                 softc->disk->d_flags |= DISKFLAG_CANDELETE;
1770                 softc->disk->d_delmaxsize = 256 * softc->params.secsize;
1771         } else
1772                 softc->disk->d_delmaxsize = maxio;
1773         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
1774                 softc->disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
1775                 softc->unmappedio = 1;
1776         }
1777         if (cpi.hba_misc & PIM_ATA_EXT)
1778                 softc->flags |= ADA_FLAG_PIM_ATA_EXT;
1779         strlcpy(softc->disk->d_descr, cgd->ident_data.model,
1780             MIN(sizeof(softc->disk->d_descr), sizeof(cgd->ident_data.model)));
1781         strlcpy(softc->disk->d_ident, cgd->ident_data.serial,
1782             MIN(sizeof(softc->disk->d_ident), sizeof(cgd->ident_data.serial)));
1783         softc->disk->d_hba_vendor = cpi.hba_vendor;
1784         softc->disk->d_hba_device = cpi.hba_device;
1785         softc->disk->d_hba_subvendor = cpi.hba_subvendor;
1786         softc->disk->d_hba_subdevice = cpi.hba_subdevice;
1787
1788         softc->disk->d_sectorsize = softc->params.secsize;
1789         softc->disk->d_mediasize = (off_t)softc->params.sectors *
1790             softc->params.secsize;
1791         if (ata_physical_sector_size(&cgd->ident_data) !=
1792             softc->params.secsize) {
1793                 softc->disk->d_stripesize =
1794                     ata_physical_sector_size(&cgd->ident_data);
1795                 softc->disk->d_stripeoffset = (softc->disk->d_stripesize -
1796                     ata_logical_sector_offset(&cgd->ident_data)) %
1797                     softc->disk->d_stripesize;
1798         } else if (softc->quirks & ADA_Q_4K) {
1799                 softc->disk->d_stripesize = 4096;
1800                 softc->disk->d_stripeoffset = 0;
1801         }
1802         softc->disk->d_fwsectors = softc->params.secs_per_track;
1803         softc->disk->d_fwheads = softc->params.heads;
1804         ata_disk_firmware_geom_adjust(softc->disk);
1805
1806         /*
1807          * Acquire a reference to the periph before we register with GEOM.
1808          * We'll release this reference once GEOM calls us back (via
1809          * adadiskgonecb()) telling us that our provider has been freed.
1810          */
1811         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
1812                 xpt_print(periph->path, "%s: lost periph during "
1813                           "registration!\n", __func__);
1814                 cam_periph_lock(periph);
1815                 return (CAM_REQ_CMP_ERR);
1816         }
1817         disk_create(softc->disk, DISK_VERSION);
1818         cam_periph_lock(periph);
1819
1820         dp = &softc->params;
1821         snprintf(announce_buf, sizeof(announce_buf),
1822             "%juMB (%ju %u byte sectors)",
1823             ((uintmax_t)dp->secsize * dp->sectors) / (1024 * 1024),
1824             (uintmax_t)dp->sectors, dp->secsize);
1825         xpt_announce_periph(periph, announce_buf);
1826         xpt_announce_quirks(periph, softc->quirks, ADA_Q_BIT_STRING);
1827
1828         /*
1829          * Create our sysctl variables, now that we know
1830          * we have successfully attached.
1831          */
1832         if (cam_periph_acquire(periph) == CAM_REQ_CMP)
1833                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
1834
1835         /*
1836          * Add async callbacks for bus reset and
1837          * bus device reset calls.  I don't bother
1838          * checking if this fails as, in most cases,
1839          * the system will function just fine without
1840          * them and the only alternative would be to
1841          * not attach the device on failure.
1842          */
1843         xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE |
1844             AC_GETDEV_CHANGED | AC_ADVINFO_CHANGED,
1845             adaasync, periph, periph->path);
1846
1847         /*
1848          * Schedule a periodic event to occasionally send an
1849          * ordered tag to a device.
1850          */
1851         callout_init_mtx(&softc->sendordered_c, cam_periph_mtx(periph), 0);
1852         callout_reset(&softc->sendordered_c,
1853             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
1854             adasendorderedtag, softc);
1855
1856         if (ADA_RA >= 0 && softc->flags & ADA_FLAG_CAN_RAHEAD) {
1857                 softc->state = ADA_STATE_RAHEAD;
1858         } else if (ADA_WC >= 0 && softc->flags & ADA_FLAG_CAN_WCACHE) {
1859                 softc->state = ADA_STATE_WCACHE;
1860         } else if ((softc->flags & ADA_FLAG_CAN_LOG)
1861                 && (softc->zone_mode != ADA_ZONE_NONE)) {
1862                 softc->state = ADA_STATE_LOGDIR;
1863         } else {
1864                 /*
1865                  * Nothing to probe, so we can just transition to the
1866                  * normal state.
1867                  */
1868                 adaprobedone(periph, NULL);
1869                 return(CAM_REQ_CMP);
1870         }
1871
1872         xpt_schedule(periph, CAM_PRIORITY_DEV);
1873
1874         return(CAM_REQ_CMP);
1875 }
1876
1877 static int
1878 ada_dsmtrim_req_create(struct ada_softc *softc, struct bio *bp, struct trim_request *req)
1879 {
1880         uint64_t lastlba = (uint64_t)-1;
1881         int c, lastcount = 0, off, ranges = 0;
1882
1883         bzero(req, sizeof(*req));
1884         TAILQ_INIT(&req->bps);
1885         do {
1886                 uint64_t lba = bp->bio_pblkno;
1887                 int count = bp->bio_bcount / softc->params.secsize;
1888
1889                 /* Try to extend the previous range. */
1890                 if (lba == lastlba) {
1891                         c = min(count, ATA_DSM_RANGE_MAX - lastcount);
1892                         lastcount += c;
1893                         off = (ranges - 1) * ATA_DSM_RANGE_SIZE;
1894                         req->data[off + 6] = lastcount & 0xff;
1895                         req->data[off + 7] =
1896                                 (lastcount >> 8) & 0xff;
1897                         count -= c;
1898                         lba += c;
1899                 }
1900
1901                 while (count > 0) {
1902                         c = min(count, ATA_DSM_RANGE_MAX);
1903                         off = ranges * ATA_DSM_RANGE_SIZE;
1904                         req->data[off + 0] = lba & 0xff;
1905                         req->data[off + 1] = (lba >> 8) & 0xff;
1906                         req->data[off + 2] = (lba >> 16) & 0xff;
1907                         req->data[off + 3] = (lba >> 24) & 0xff;
1908                         req->data[off + 4] = (lba >> 32) & 0xff;
1909                         req->data[off + 5] = (lba >> 40) & 0xff;
1910                         req->data[off + 6] = c & 0xff;
1911                         req->data[off + 7] = (c >> 8) & 0xff;
1912                         lba += c;
1913                         count -= c;
1914                         lastcount = c;
1915                         ranges++;
1916                         /*
1917                          * Its the caller's responsibility to ensure the
1918                          * request will fit so we don't need to check for
1919                          * overrun here
1920                          */
1921                 }
1922                 lastlba = lba;
1923                 TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1924
1925                 bp = cam_iosched_next_trim(softc->cam_iosched);
1926                 if (bp == NULL)
1927                         break;
1928                 if (bp->bio_bcount / softc->params.secsize >
1929                     (softc->trim_max_ranges - ranges) * ATA_DSM_RANGE_MAX) {
1930                         cam_iosched_put_back_trim(softc->cam_iosched, bp);
1931                         break;
1932                 }
1933         } while (1);
1934
1935         return (ranges);
1936 }
1937
1938 static void
1939 ada_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1940 {
1941         struct trim_request *req = &softc->trim_req;
1942         int ranges;
1943
1944         ranges = ada_dsmtrim_req_create(softc, bp, req);
1945         cam_fill_ataio(ataio,
1946             ada_retry_count,
1947             adadone,
1948             CAM_DIR_OUT,
1949             0,
1950             req->data,
1951             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1952             ada_default_timeout * 1000);
1953         ata_48bit_cmd(ataio, ATA_DATA_SET_MANAGEMENT,
1954             ATA_DSM_TRIM, 0, howmany(ranges, ATA_DSM_BLK_RANGES));
1955 }
1956
1957 static void
1958 ada_ncq_dsmtrim(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1959 {
1960         struct trim_request *req = &softc->trim_req;
1961         int ranges;
1962
1963         ranges = ada_dsmtrim_req_create(softc, bp, req);
1964         cam_fill_ataio(ataio,
1965             ada_retry_count,
1966             adadone,
1967             CAM_DIR_OUT,
1968             0,
1969             req->data,
1970             howmany(ranges, ATA_DSM_BLK_RANGES) * ATA_DSM_BLK_SIZE,
1971             ada_default_timeout * 1000);
1972         ata_ncq_cmd(ataio,
1973             ATA_SEND_FPDMA_QUEUED,
1974             0,
1975             howmany(ranges, ATA_DSM_BLK_RANGES));
1976         ataio->cmd.sector_count_exp = ATA_SFPDMA_DSM;
1977         ataio->ata_flags |= ATA_FLAG_AUX;
1978         ataio->aux = 1;
1979 }
1980
1981 static void
1982 ada_cfaerase(struct ada_softc *softc, struct bio *bp, struct ccb_ataio *ataio)
1983 {
1984         struct trim_request *req = &softc->trim_req;
1985         uint64_t lba = bp->bio_pblkno;
1986         uint16_t count = bp->bio_bcount / softc->params.secsize;
1987
1988         bzero(req, sizeof(*req));
1989         TAILQ_INIT(&req->bps);
1990         TAILQ_INSERT_TAIL(&req->bps, bp, bio_queue);
1991
1992         cam_fill_ataio(ataio,
1993             ada_retry_count,
1994             adadone,
1995             CAM_DIR_NONE,
1996             0,
1997             NULL,
1998             0,
1999             ada_default_timeout*1000);
2000
2001         if (count >= 256)
2002                 count = 0;
2003         ata_28bit_cmd(ataio, ATA_CFA_ERASE, 0, lba, count);
2004 }
2005
2006 static int
2007 ada_zone_bio_to_ata(int disk_zone_cmd)
2008 {
2009         switch (disk_zone_cmd) {
2010         case DISK_ZONE_OPEN:
2011                 return ATA_ZM_OPEN_ZONE;
2012         case DISK_ZONE_CLOSE:
2013                 return ATA_ZM_CLOSE_ZONE;
2014         case DISK_ZONE_FINISH:
2015                 return ATA_ZM_FINISH_ZONE;
2016         case DISK_ZONE_RWP:
2017                 return ATA_ZM_RWP;
2018         }
2019
2020         return -1;
2021 }
2022
2023 static int
2024 ada_zone_cmd(struct cam_periph *periph, union ccb *ccb, struct bio *bp,
2025              int *queue_ccb)
2026 {
2027         struct ada_softc *softc;
2028         int error;
2029
2030         error = 0;
2031
2032         if (bp->bio_cmd != BIO_ZONE) {
2033                 error = EINVAL;
2034                 goto bailout;
2035         }
2036
2037         softc = periph->softc;
2038
2039         switch (bp->bio_zone.zone_cmd) {
2040         case DISK_ZONE_OPEN:
2041         case DISK_ZONE_CLOSE:
2042         case DISK_ZONE_FINISH:
2043         case DISK_ZONE_RWP: {
2044                 int zone_flags;
2045                 int zone_sa;
2046                 uint64_t lba;
2047
2048                 zone_sa = ada_zone_bio_to_ata(bp->bio_zone.zone_cmd);
2049                 if (zone_sa == -1) {
2050                         xpt_print(periph->path, "Cannot translate zone "
2051                             "cmd %#x to ATA\n", bp->bio_zone.zone_cmd);
2052                         error = EINVAL;
2053                         goto bailout;
2054                 }
2055
2056                 zone_flags = 0;
2057                 lba = bp->bio_zone.zone_params.rwp.id;
2058
2059                 if (bp->bio_zone.zone_params.rwp.flags &
2060                     DISK_ZONE_RWP_FLAG_ALL)
2061                         zone_flags |= ZBC_OUT_ALL;
2062
2063                 ata_zac_mgmt_out(&ccb->ataio,
2064                                  /*retries*/ ada_retry_count,
2065                                  /*cbfcnp*/ adadone,
2066                                  /*use_ncq*/ (softc->flags &
2067                                               ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2068                                  /*zm_action*/ zone_sa,
2069                                  /*zone_id*/ lba,
2070                                  /*zone_flags*/ zone_flags,
2071                                  /*sector_count*/ 0,
2072                                  /*data_ptr*/ NULL,
2073                                  /*dxfer_len*/ 0,
2074                                  /*timeout*/ ada_default_timeout * 1000);
2075                 *queue_ccb = 1;
2076
2077                 break;
2078         }
2079         case DISK_ZONE_REPORT_ZONES: {
2080                 uint8_t *rz_ptr;
2081                 uint32_t num_entries, alloc_size;
2082                 struct disk_zone_report *rep;
2083
2084                 rep = &bp->bio_zone.zone_params.report;
2085
2086                 num_entries = rep->entries_allocated;
2087                 if (num_entries == 0) {
2088                         xpt_print(periph->path, "No entries allocated for "
2089                             "Report Zones request\n");
2090                         error = EINVAL;
2091                         goto bailout;
2092                 }
2093                 alloc_size = sizeof(struct scsi_report_zones_hdr) +
2094                     (sizeof(struct scsi_report_zones_desc) * num_entries);
2095                 alloc_size = min(alloc_size, softc->disk->d_maxsize);
2096                 rz_ptr = malloc(alloc_size, M_ATADA, M_NOWAIT | M_ZERO);
2097                 if (rz_ptr == NULL) {
2098                         xpt_print(periph->path, "Unable to allocate memory "
2099                            "for Report Zones request\n");
2100                         error = ENOMEM;
2101                         goto bailout;
2102                 }
2103
2104                 ata_zac_mgmt_in(&ccb->ataio,
2105                                 /*retries*/ ada_retry_count,
2106                                 /*cbcfnp*/ adadone,
2107                                 /*use_ncq*/ (softc->flags &
2108                                              ADA_FLAG_PIM_ATA_EXT) ? 1 : 0,
2109                                 /*zm_action*/ ATA_ZM_REPORT_ZONES,
2110                                 /*zone_id*/ rep->starting_id,
2111                                 /*zone_flags*/ rep->rep_options,
2112                                 /*data_ptr*/ rz_ptr,
2113                                 /*dxfer_len*/ alloc_size,
2114                                 /*timeout*/ ada_default_timeout * 1000);
2115
2116                 /*
2117                  * For BIO_ZONE, this isn't normally needed.  However, it
2118                  * is used by devstat_end_transaction_bio() to determine
2119                  * how much data was transferred.
2120                  */
2121                 /*
2122                  * XXX KDM we have a problem.  But I'm not sure how to fix
2123                  * it.  devstat uses bio_bcount - bio_resid to calculate
2124                  * the amount of data transferred.   The GEOM disk code
2125                  * uses bio_length - bio_resid to calculate the amount of
2126                  * data in bio_completed.  We have different structure
2127                  * sizes above and below the ada(4) driver.  So, if we
2128                  * use the sizes above, the amount transferred won't be
2129                  * quite accurate for devstat.  If we use different sizes
2130                  * for bio_bcount and bio_length (above and below
2131                  * respectively), then the residual needs to match one or
2132                  * the other.  Everything is calculated after the bio
2133                  * leaves the driver, so changing the values around isn't
2134                  * really an option.  For now, just set the count to the
2135                  * passed in length.  This means that the calculations
2136                  * above (e.g. bio_completed) will be correct, but the
2137                  * amount of data reported to devstat will be slightly
2138                  * under or overstated.
2139                  */
2140                 bp->bio_bcount = bp->bio_length;
2141
2142                 *queue_ccb = 1;
2143
2144                 break;
2145         }
2146         case DISK_ZONE_GET_PARAMS: {
2147                 struct disk_zone_disk_params *params;
2148
2149                 params = &bp->bio_zone.zone_params.disk_params;
2150                 bzero(params, sizeof(*params));
2151
2152                 switch (softc->zone_mode) {
2153                 case ADA_ZONE_DRIVE_MANAGED:
2154                         params->zone_mode = DISK_ZONE_MODE_DRIVE_MANAGED;
2155                         break;
2156                 case ADA_ZONE_HOST_AWARE:
2157                         params->zone_mode = DISK_ZONE_MODE_HOST_AWARE;
2158                         break;
2159                 case ADA_ZONE_HOST_MANAGED:
2160                         params->zone_mode = DISK_ZONE_MODE_HOST_MANAGED;
2161                         break;
2162                 default:
2163                 case ADA_ZONE_NONE:
2164                         params->zone_mode = DISK_ZONE_MODE_NONE;
2165                         break;
2166                 }
2167
2168                 if (softc->zone_flags & ADA_ZONE_FLAG_URSWRZ)
2169                         params->flags |= DISK_ZONE_DISK_URSWRZ;
2170
2171                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_SEQ_SET) {
2172                         params->optimal_seq_zones = softc->optimal_seq_zones;
2173                         params->flags |= DISK_ZONE_OPT_SEQ_SET;
2174                 }
2175
2176                 if (softc->zone_flags & ADA_ZONE_FLAG_OPT_NONSEQ_SET) {
2177                         params->optimal_nonseq_zones =
2178                             softc->optimal_nonseq_zones;
2179                         params->flags |= DISK_ZONE_OPT_NONSEQ_SET;
2180                 }
2181
2182                 if (softc->zone_flags & ADA_ZONE_FLAG_MAX_SEQ_SET) {
2183                         params->max_seq_zones = softc->max_seq_zones;
2184                         params->flags |= DISK_ZONE_MAX_SEQ_SET;
2185                 }
2186                 if (softc->zone_flags & ADA_ZONE_FLAG_RZ_SUP)
2187                         params->flags |= DISK_ZONE_RZ_SUP;
2188
2189                 if (softc->zone_flags & ADA_ZONE_FLAG_OPEN_SUP)
2190                         params->flags |= DISK_ZONE_OPEN_SUP;
2191
2192                 if (softc->zone_flags & ADA_ZONE_FLAG_CLOSE_SUP)
2193                         params->flags |= DISK_ZONE_CLOSE_SUP;
2194
2195                 if (softc->zone_flags & ADA_ZONE_FLAG_FINISH_SUP)
2196                         params->flags |= DISK_ZONE_FINISH_SUP;
2197
2198                 if (softc->zone_flags & ADA_ZONE_FLAG_RWP_SUP)
2199                         params->flags |= DISK_ZONE_RWP_SUP;
2200                 break;
2201         }
2202         default:
2203                 break;
2204         }
2205 bailout:
2206         return (error);
2207 }
2208
2209 static void
2210 adastart(struct cam_periph *periph, union ccb *start_ccb)
2211 {
2212         struct ada_softc *softc = (struct ada_softc *)periph->softc;
2213         struct ccb_ataio *ataio = &start_ccb->ataio;
2214
2215         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("adastart\n"));
2216
2217         switch (softc->state) {
2218         case ADA_STATE_NORMAL:
2219         {
2220                 struct bio *bp;
2221                 u_int8_t tag_code;
2222
2223                 bp = cam_iosched_next_bio(softc->cam_iosched);
2224                 if (bp == NULL) {
2225                         xpt_release_ccb(start_ccb);
2226                         break;
2227                 }
2228
2229                 if ((bp->bio_flags & BIO_ORDERED) != 0 ||
2230                     (bp->bio_cmd != BIO_DELETE && (softc->flags & ADA_FLAG_NEED_OTAG) != 0)) {
2231                         softc->flags &= ~ADA_FLAG_NEED_OTAG;
2232                         softc->flags |= ADA_FLAG_WAS_OTAG;
2233                         tag_code = 0;
2234                 } else {
2235                         tag_code = 1;
2236                 }
2237                 switch (bp->bio_cmd) {
2238                 case BIO_WRITE:
2239                 case BIO_READ:
2240                 {
2241                         uint64_t lba = bp->bio_pblkno;
2242                         uint16_t count = bp->bio_bcount / softc->params.secsize;
2243                         void *data_ptr;
2244                         int rw_op;
2245
2246                         if (bp->bio_cmd == BIO_WRITE) {
2247                                 softc->flags |= ADA_FLAG_DIRTY;
2248                                 rw_op = CAM_DIR_OUT;
2249                         } else {
2250                                 rw_op = CAM_DIR_IN;
2251                         }
2252
2253                         data_ptr = bp->bio_data;
2254                         if ((bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0) {
2255                                 rw_op |= CAM_DATA_BIO;
2256                                 data_ptr = bp;
2257                         }
2258
2259 #ifdef ADA_TEST_FAILURE
2260                         int fail = 0;
2261
2262                         /*
2263                          * Support the failure ioctls.  If the command is a
2264                          * read, and there are pending forced read errors, or
2265                          * if a write and pending write errors, then fail this
2266                          * operation with EIO.  This is useful for testing
2267                          * purposes.  Also, support having every Nth read fail.
2268                          *
2269                          * This is a rather blunt tool.
2270                          */
2271                         if (bp->bio_cmd == BIO_READ) {
2272                                 if (softc->force_read_error) {
2273                                         softc->force_read_error--;
2274                                         fail = 1;
2275                                 }
2276                                 if (softc->periodic_read_error > 0) {
2277                                         if (++softc->periodic_read_count >=
2278                                             softc->periodic_read_error) {
2279                                                 softc->periodic_read_count = 0;
2280                                                 fail = 1;
2281                                         }
2282                                 }
2283                         } else {
2284                                 if (softc->force_write_error) {
2285                                         softc->force_write_error--;
2286                                         fail = 1;
2287                                 }
2288                         }
2289                         if (fail) {
2290                                 biofinish(bp, NULL, EIO);
2291                                 xpt_release_ccb(start_ccb);
2292                                 adaschedule(periph);
2293                                 return;
2294                         }
2295 #endif
2296                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
2297                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
2298                             PAGE_SIZE == bp->bio_ma_n,
2299                             ("Short bio %p", bp));
2300                         cam_fill_ataio(ataio,
2301                             ada_retry_count,
2302                             adadone,
2303                             rw_op,
2304                             0,
2305                             data_ptr,
2306                             bp->bio_bcount,
2307                             ada_default_timeout*1000);
2308
2309                         if ((softc->flags & ADA_FLAG_CAN_NCQ) && tag_code) {
2310                                 if (bp->bio_cmd == BIO_READ) {
2311                                         ata_ncq_cmd(ataio, ATA_READ_FPDMA_QUEUED,
2312                                             lba, count);
2313                                 } else {
2314                                         ata_ncq_cmd(ataio, ATA_WRITE_FPDMA_QUEUED,
2315                                             lba, count);
2316                                 }
2317                         } else if ((softc->flags & ADA_FLAG_CAN_48BIT) &&
2318                             (lba + count >= ATA_MAX_28BIT_LBA ||
2319                             count > 256)) {
2320                                 if (softc->flags & ADA_FLAG_CAN_DMA48) {
2321                                         if (bp->bio_cmd == BIO_READ) {
2322                                                 ata_48bit_cmd(ataio, ATA_READ_DMA48,
2323                                                     0, lba, count);
2324                                         } else {
2325                                                 ata_48bit_cmd(ataio, ATA_WRITE_DMA48,
2326                                                     0, lba, count);
2327                                         }
2328                                 } else {
2329                                         if (bp->bio_cmd == BIO_READ) {
2330                                                 ata_48bit_cmd(ataio, ATA_READ_MUL48,
2331                                                     0, lba, count);
2332                                         } else {
2333                                                 ata_48bit_cmd(ataio, ATA_WRITE_MUL48,
2334                                                     0, lba, count);
2335                                         }
2336                                 }
2337                         } else {
2338                                 if (count == 256)
2339                                         count = 0;
2340                                 if (softc->flags & ADA_FLAG_CAN_DMA) {
2341                                         if (bp->bio_cmd == BIO_READ) {
2342                                                 ata_28bit_cmd(ataio, ATA_READ_DMA,
2343                                                     0, lba, count);
2344                                         } else {
2345                                                 ata_28bit_cmd(ataio, ATA_WRITE_DMA,
2346                                                     0, lba, count);
2347                                         }
2348                                 } else {
2349                                         if (bp->bio_cmd == BIO_READ) {
2350                                                 ata_28bit_cmd(ataio, ATA_READ_MUL,
2351                                                     0, lba, count);
2352                                         } else {
2353                                                 ata_28bit_cmd(ataio, ATA_WRITE_MUL,
2354                                                     0, lba, count);
2355                                         }
2356                                 }
2357                         }
2358                         break;
2359                 }
2360                 case BIO_DELETE:
2361                         switch (softc->delete_method) {
2362                         case ADA_DELETE_NCQ_DSM_TRIM:
2363                                 ada_ncq_dsmtrim(softc, bp, ataio);
2364                                 break;
2365                         case ADA_DELETE_DSM_TRIM:
2366                                 ada_dsmtrim(softc, bp, ataio);
2367                                 break;
2368                         case ADA_DELETE_CFA_ERASE:
2369                                 ada_cfaerase(softc, bp, ataio);
2370                                 break;
2371                         default:
2372                                 biofinish(bp, NULL, EOPNOTSUPP);
2373                                 xpt_release_ccb(start_ccb);
2374                                 adaschedule(periph);
2375                                 return;
2376                         }
2377                         start_ccb->ccb_h.ccb_state = ADA_CCB_TRIM;
2378                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2379                         cam_iosched_submit_trim(softc->cam_iosched);
2380                         goto out;
2381                 case BIO_FLUSH:
2382                         cam_fill_ataio(ataio,
2383                             1,
2384                             adadone,
2385                             CAM_DIR_NONE,
2386                             0,
2387                             NULL,
2388                             0,
2389                             ada_default_timeout*1000);
2390
2391                         if (softc->flags & ADA_FLAG_CAN_48BIT)
2392                                 ata_48bit_cmd(ataio, ATA_FLUSHCACHE48, 0, 0, 0);
2393                         else
2394                                 ata_28bit_cmd(ataio, ATA_FLUSHCACHE, 0, 0, 0);
2395                         break;
2396                 case BIO_ZONE: {
2397                         int error, queue_ccb;
2398
2399                         queue_ccb = 0;
2400
2401                         error = ada_zone_cmd(periph, start_ccb, bp, &queue_ccb);
2402                         if ((error != 0)
2403                          || (queue_ccb == 0)) {
2404                                 biofinish(bp, NULL, error);
2405                                 xpt_release_ccb(start_ccb);
2406                                 return;
2407                         }
2408                         break;
2409                 }
2410                 }
2411                 start_ccb->ccb_h.ccb_state = ADA_CCB_BUFFER_IO;
2412                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
2413 out:
2414                 start_ccb->ccb_h.ccb_bp = bp;
2415                 softc->outstanding_cmds++;
2416                 softc->refcount++;
2417                 cam_periph_unlock(periph);
2418                 xpt_action(start_ccb);
2419                 cam_periph_lock(periph);
2420                 softc->refcount--;
2421
2422                 /* May have more work to do, so ensure we stay scheduled */
2423                 adaschedule(periph);
2424                 break;
2425         }
2426         case ADA_STATE_RAHEAD:
2427         case ADA_STATE_WCACHE:
2428         {
2429                 cam_fill_ataio(ataio,
2430                     1,
2431                     adadone,
2432                     CAM_DIR_NONE,
2433                     0,
2434                     NULL,
2435                     0,
2436                     ada_default_timeout*1000);
2437
2438                 if (softc->state == ADA_STATE_RAHEAD) {
2439                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_RA ?
2440                             ATA_SF_ENAB_RCACHE : ATA_SF_DIS_RCACHE, 0, 0);
2441                         start_ccb->ccb_h.ccb_state = ADA_CCB_RAHEAD;
2442                 } else {
2443                         ata_28bit_cmd(ataio, ATA_SETFEATURES, ADA_WC ?
2444                             ATA_SF_ENAB_WCACHE : ATA_SF_DIS_WCACHE, 0, 0);
2445                         start_ccb->ccb_h.ccb_state = ADA_CCB_WCACHE;
2446                 }
2447                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2448                 xpt_action(start_ccb);
2449                 break;
2450         }
2451         case ADA_STATE_LOGDIR:
2452         {
2453                 struct ata_gp_log_dir *log_dir;
2454
2455                 if ((softc->flags & ADA_FLAG_CAN_LOG) == 0) {
2456                         adaprobedone(periph, start_ccb);
2457                         break;
2458                 }
2459
2460                 log_dir = malloc(sizeof(*log_dir), M_ATADA, M_NOWAIT|M_ZERO);
2461                 if (log_dir == NULL) {
2462                         xpt_print(periph->path, "Couldn't malloc log_dir "
2463                             "data\n");
2464                         softc->state = ADA_STATE_NORMAL;
2465                         xpt_release_ccb(start_ccb);
2466                         break;
2467                 }
2468
2469
2470                 ata_read_log(ataio,
2471                     /*retries*/1,
2472                     /*cbfcnp*/adadone,
2473                     /*log_address*/ ATA_LOG_DIRECTORY,
2474                     /*page_number*/ 0,
2475                     /*block_count*/ 1,
2476                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2477                                  CAM_ATAIO_DMA : 0,
2478                     /*data_ptr*/ (uint8_t *)log_dir,
2479                     /*dxfer_len*/sizeof(*log_dir),
2480                     /*timeout*/ada_default_timeout*1000);
2481
2482                 start_ccb->ccb_h.ccb_state = ADA_CCB_LOGDIR;
2483                 xpt_action(start_ccb);
2484                 break;
2485         }
2486         case ADA_STATE_IDDIR:
2487         {
2488                 struct ata_identify_log_pages *id_dir;
2489
2490                 id_dir = malloc(sizeof(*id_dir), M_ATADA, M_NOWAIT | M_ZERO);
2491                 if (id_dir == NULL) {
2492                         xpt_print(periph->path, "Couldn't malloc id_dir "
2493                             "data\n");
2494                         adaprobedone(periph, start_ccb);
2495                         break;
2496                 }
2497
2498                 ata_read_log(ataio,
2499                     /*retries*/1,
2500                     /*cbfcnp*/adadone,
2501                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2502                     /*page_number*/ ATA_IDL_PAGE_LIST,
2503                     /*block_count*/ 1,
2504                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2505                                  CAM_ATAIO_DMA : 0,
2506                     /*data_ptr*/ (uint8_t *)id_dir,
2507                     /*dxfer_len*/ sizeof(*id_dir),
2508                     /*timeout*/ada_default_timeout*1000);
2509
2510                 start_ccb->ccb_h.ccb_state = ADA_CCB_IDDIR;
2511                 xpt_action(start_ccb);
2512                 break;
2513         }
2514         case ADA_STATE_SUP_CAP:
2515         {
2516                 struct ata_identify_log_sup_cap *sup_cap;
2517
2518                 sup_cap = malloc(sizeof(*sup_cap), M_ATADA, M_NOWAIT|M_ZERO);
2519                 if (sup_cap == NULL) {
2520                         xpt_print(periph->path, "Couldn't malloc sup_cap "
2521                             "data\n");
2522                         adaprobedone(periph, start_ccb);
2523                         break;
2524                 }
2525
2526                 ata_read_log(ataio,
2527                     /*retries*/1,
2528                     /*cbfcnp*/adadone,
2529                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2530                     /*page_number*/ ATA_IDL_SUP_CAP,
2531                     /*block_count*/ 1,
2532                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2533                                  CAM_ATAIO_DMA : 0,
2534                     /*data_ptr*/ (uint8_t *)sup_cap,
2535                     /*dxfer_len*/ sizeof(*sup_cap),
2536                     /*timeout*/ada_default_timeout*1000);
2537
2538                 start_ccb->ccb_h.ccb_state = ADA_CCB_SUP_CAP;
2539                 xpt_action(start_ccb);
2540                 break;
2541         }
2542         case ADA_STATE_ZONE:
2543         {
2544                 struct ata_zoned_info_log *ata_zone;
2545
2546                 ata_zone = malloc(sizeof(*ata_zone), M_ATADA, M_NOWAIT|M_ZERO);
2547                 if (ata_zone == NULL) {
2548                         xpt_print(periph->path, "Couldn't malloc ata_zone "
2549                             "data\n");
2550                         adaprobedone(periph, start_ccb);
2551                         break;
2552                 }
2553
2554                 ata_read_log(ataio,
2555                     /*retries*/1,
2556                     /*cbfcnp*/adadone,
2557                     /*log_address*/ ATA_IDENTIFY_DATA_LOG,
2558                     /*page_number*/ ATA_IDL_ZDI,
2559                     /*block_count*/ 1,
2560                     /*protocol*/ softc->flags & ADA_FLAG_CAN_DMA ?
2561                                  CAM_ATAIO_DMA : 0,
2562                     /*data_ptr*/ (uint8_t *)ata_zone,
2563                     /*dxfer_len*/ sizeof(*ata_zone),
2564                     /*timeout*/ada_default_timeout*1000);
2565
2566                 start_ccb->ccb_h.ccb_state = ADA_CCB_ZONE;
2567                 xpt_action(start_ccb);
2568                 break;
2569         }
2570         }
2571 }
2572
2573 static void
2574 adaprobedone(struct cam_periph *periph, union ccb *ccb)
2575 {
2576         struct ada_softc *softc;
2577
2578         softc = (struct ada_softc *)periph->softc;
2579
2580         if (ccb != NULL)
2581                 xpt_release_ccb(ccb);
2582
2583         softc->state = ADA_STATE_NORMAL;
2584         softc->flags |= ADA_FLAG_PROBED;
2585         adaschedule(periph);
2586         if ((softc->flags & ADA_FLAG_ANNOUNCED) == 0) {
2587                 softc->flags |= ADA_FLAG_ANNOUNCED;
2588                 cam_periph_unhold(periph);
2589         } else {
2590                 cam_periph_release_locked(periph);
2591         }
2592 }
2593
2594 static void
2595 adazonedone(struct cam_periph *periph, union ccb *ccb)
2596 {
2597         struct ada_softc *softc;
2598         struct bio *bp;
2599
2600         softc = periph->softc;
2601         bp = (struct bio *)ccb->ccb_h.ccb_bp;
2602
2603         switch (bp->bio_zone.zone_cmd) {
2604         case DISK_ZONE_OPEN:
2605         case DISK_ZONE_CLOSE:
2606         case DISK_ZONE_FINISH:
2607         case DISK_ZONE_RWP:
2608                 break;
2609         case DISK_ZONE_REPORT_ZONES: {
2610                 uint32_t avail_len;
2611                 struct disk_zone_report *rep;
2612                 struct scsi_report_zones_hdr *hdr;
2613                 struct scsi_report_zones_desc *desc;
2614                 struct disk_zone_rep_entry *entry;
2615                 uint32_t num_alloced, hdr_len, num_avail;
2616                 uint32_t num_to_fill, i;
2617
2618                 rep = &bp->bio_zone.zone_params.report;
2619                 avail_len = ccb->ataio.dxfer_len - ccb->ataio.resid;
2620                 /*
2621                  * Note that bio_resid isn't normally used for zone
2622                  * commands, but it is used by devstat_end_transaction_bio()
2623                  * to determine how much data was transferred.  Because
2624                  * the size of the SCSI/ATA data structures is different
2625                  * than the size of the BIO interface structures, the
2626                  * amount of data actually transferred from the drive will
2627                  * be different than the amount of data transferred to
2628                  * the user.
2629                  */
2630                 num_alloced = rep->entries_allocated;
2631                 hdr = (struct scsi_report_zones_hdr *)ccb->ataio.data_ptr;
2632                 if (avail_len < sizeof(*hdr)) {
2633                         /*
2634                          * Is there a better error than EIO here?  We asked
2635                          * for at least the header, and we got less than
2636                          * that.
2637                          */
2638                         bp->bio_error = EIO;
2639                         bp->bio_flags |= BIO_ERROR;
2640                         bp->bio_resid = bp->bio_bcount;
2641                         break;
2642                 }
2643
2644                 hdr_len = le32dec(hdr->length);
2645                 if (hdr_len > 0)
2646                         rep->entries_available = hdr_len / sizeof(*desc);
2647                 else
2648                         rep->entries_available = 0;
2649                 /*
2650                  * NOTE: using the same values for the BIO version of the
2651                  * same field as the SCSI/ATA values.  This means we could
2652                  * get some additional values that aren't defined in bio.h
2653                  * if more values of the same field are defined later.
2654                  */
2655                 rep->header.same = hdr->byte4 & SRZ_SAME_MASK;
2656                 rep->header.maximum_lba = le64dec(hdr->maximum_lba);
2657                 /*
2658                  * If the drive reports no entries that match the query,
2659                  * we're done.
2660                  */
2661                 if (hdr_len == 0) {
2662                         rep->entries_filled = 0;
2663                         bp->bio_resid = bp->bio_bcount;
2664                         break;
2665                 }
2666
2667                 num_avail = min((avail_len - sizeof(*hdr)) / sizeof(*desc),
2668                                 hdr_len / sizeof(*desc));
2669                 /*
2670                  * If the drive didn't return any data, then we're done.
2671                  */
2672                 if (num_avail == 0) {
2673                         rep->entries_filled = 0;
2674                         bp->bio_resid = bp->bio_bcount;
2675                         break;
2676                 }
2677
2678                 num_to_fill = min(num_avail, rep->entries_allocated);
2679                 /*
2680                  * If the user didn't allocate any entries for us to fill,
2681                  * we're done.
2682                  */
2683                 if (num_to_fill == 0) {
2684                         rep->entries_filled = 0;
2685                         bp->bio_resid = bp->bio_bcount;
2686                         break;
2687                 }
2688
2689                 for (i = 0, desc = &hdr->desc_list[0], entry=&rep->entries[0];
2690                      i < num_to_fill; i++, desc++, entry++) {
2691                         /*
2692                          * NOTE: we're mapping the values here directly
2693                          * from the SCSI/ATA bit definitions to the bio.h
2694                          * definitions.  There is also a warning in
2695                          * disk_zone.h, but the impact is that if
2696                          * additional values are added in the SCSI/ATA
2697                          * specs these will be visible to consumers of
2698                          * this interface.
2699                          */
2700                         entry->zone_type = desc->zone_type & SRZ_TYPE_MASK;
2701                         entry->zone_condition =
2702                             (desc->zone_flags & SRZ_ZONE_COND_MASK) >>
2703                             SRZ_ZONE_COND_SHIFT;
2704                         entry->zone_flags |= desc->zone_flags &
2705                             (SRZ_ZONE_NON_SEQ|SRZ_ZONE_RESET);
2706                         entry->zone_length = le64dec(desc->zone_length);
2707                         entry->zone_start_lba = le64dec(desc->zone_start_lba);
2708                         entry->write_pointer_lba =
2709                             le64dec(desc->write_pointer_lba);
2710                 }
2711                 rep->entries_filled = num_to_fill;
2712                 /*
2713                  * Note that this residual is accurate from the user's
2714                  * standpoint, but the amount transferred isn't accurate
2715                  * from the standpoint of what actually came back from the
2716                  * drive.
2717                  */
2718                 bp->bio_resid = bp->bio_bcount - (num_to_fill * sizeof(*entry));
2719                 break;
2720         }
2721         case DISK_ZONE_GET_PARAMS:
2722         default:
2723                 /*
2724                  * In theory we should not get a GET_PARAMS bio, since it
2725                  * should be handled without queueing the command to the
2726                  * drive.
2727                  */
2728                 panic("%s: Invalid zone command %d", __func__,
2729                     bp->bio_zone.zone_cmd);
2730                 break;
2731         }
2732
2733         if (bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES)
2734                 free(ccb->ataio.data_ptr, M_ATADA);
2735 }
2736
2737
2738 static void
2739 adadone(struct cam_periph *periph, union ccb *done_ccb)
2740 {
2741         struct ada_softc *softc;
2742         struct ccb_ataio *ataio;
2743         struct cam_path *path;
2744         uint32_t priority;
2745         int state;
2746
2747         softc = (struct ada_softc *)periph->softc;
2748         ataio = &done_ccb->ataio;
2749         path = done_ccb->ccb_h.path;
2750         priority = done_ccb->ccb_h.pinfo.priority;
2751
2752         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("adadone\n"));
2753
2754         state = ataio->ccb_h.ccb_state & ADA_CCB_TYPE_MASK;
2755         switch (state) {
2756         case ADA_CCB_BUFFER_IO:
2757         case ADA_CCB_TRIM:
2758         {
2759                 struct bio *bp;
2760                 int error;
2761
2762                 cam_periph_lock(periph);
2763                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
2764                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2765                         error = adaerror(done_ccb, 0, 0);
2766                         if (error == ERESTART) {
2767                                 /* A retry was scheduled, so just return. */
2768                                 cam_periph_unlock(periph);
2769                                 return;
2770                         }
2771                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2772                                 cam_release_devq(path,
2773                                                  /*relsim_flags*/0,
2774                                                  /*reduction*/0,
2775                                                  /*timeout*/0,
2776                                                  /*getcount_only*/0);
2777                         /*
2778                          * If we get an error on an NCQ DSM TRIM, fall back
2779                          * to a non-NCQ DSM TRIM forever. Please note that if
2780                          * CAN_NCQ_TRIM is set, CAN_TRIM is necessarily set too.
2781                          * However, for this one trim, we treat it as advisory
2782                          * and return success up the stack.
2783                          */
2784                         if (state == ADA_CCB_TRIM &&
2785                             error != 0 &&
2786                             (softc->flags & ADA_FLAG_CAN_NCQ_TRIM) != 0) {
2787                                 softc->flags &= ~ADA_FLAG_CAN_NCQ_TRIM;
2788                                 error = 0;
2789                                 adasetdeletemethod(softc);
2790                         }
2791                 } else {
2792                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
2793                                 panic("REQ_CMP with QFRZN");
2794
2795                         error = 0;
2796                 }
2797                 bp->bio_error = error;
2798                 if (error != 0) {
2799                         bp->bio_resid = bp->bio_bcount;
2800                         bp->bio_flags |= BIO_ERROR;
2801                 } else {
2802                         if (bp->bio_cmd == BIO_ZONE)
2803                                 adazonedone(periph, done_ccb);
2804                         else if (state == ADA_CCB_TRIM)
2805                                 bp->bio_resid = 0;
2806                         else
2807                                 bp->bio_resid = ataio->resid;
2808
2809                         if ((bp->bio_resid > 0)
2810                          && (bp->bio_cmd != BIO_ZONE))
2811                                 bp->bio_flags |= BIO_ERROR;
2812                 }
2813                 softc->outstanding_cmds--;
2814                 if (softc->outstanding_cmds == 0)
2815                         softc->flags |= ADA_FLAG_WAS_OTAG;
2816
2817                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
2818                 xpt_release_ccb(done_ccb);
2819                 if (state == ADA_CCB_TRIM) {
2820                         TAILQ_HEAD(, bio) queue;
2821                         struct bio *bp1;
2822
2823                         TAILQ_INIT(&queue);
2824                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
2825                         /*
2826                          * Normally, the xpt_release_ccb() above would make sure
2827                          * that when we have more work to do, that work would
2828                          * get kicked off. However, we specifically keep
2829                          * trim_running set to 0 before the call above to allow
2830                          * other I/O to progress when many BIO_DELETE requests
2831                          * are pushed down. We set trim_running to 0 and call
2832                          * daschedule again so that we don't stall if there are
2833                          * no other I/Os pending apart from BIO_DELETEs.
2834                          */
2835                         cam_iosched_trim_done(softc->cam_iosched);
2836                         adaschedule(periph);
2837                         cam_periph_unlock(periph);
2838                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
2839                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
2840                                 bp1->bio_error = error;
2841                                 if (error != 0) {
2842                                         bp1->bio_flags |= BIO_ERROR;
2843                                         bp1->bio_resid = bp1->bio_bcount;
2844                                 } else
2845                                         bp1->bio_resid = 0;
2846                                 biodone(bp1);
2847                         }
2848                 } else {
2849                         adaschedule(periph);
2850                         cam_periph_unlock(periph);
2851                         biodone(bp);
2852                 }
2853                 return;
2854         }
2855         case ADA_CCB_RAHEAD:
2856         {
2857                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2858                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2859                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2860                                 cam_release_devq(path, 0, 0, 0, FALSE);
2861                                 return;
2862                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2863                                 cam_release_devq(path,
2864                                     /*relsim_flags*/0,
2865                                     /*reduction*/0,
2866                                     /*timeout*/0,
2867                                     /*getcount_only*/0);
2868                         }
2869                 }
2870
2871                 /*
2872                  * Since our peripheral may be invalidated by an error
2873                  * above or an external event, we must release our CCB
2874                  * before releasing the reference on the peripheral.
2875                  * The peripheral will only go away once the last reference
2876                  * is removed, and we need it around for the CCB release
2877                  * operation.
2878                  */
2879
2880                 xpt_release_ccb(done_ccb);
2881                 softc->state = ADA_STATE_WCACHE;
2882                 xpt_schedule(periph, priority);
2883                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2884                 cam_release_devq(path, 0, 0, 0, FALSE);
2885                 return;
2886         }
2887         case ADA_CCB_WCACHE:
2888         {
2889                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2890                         if (adaerror(done_ccb, 0, 0) == ERESTART) {
2891                                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2892                                 cam_release_devq(path, 0, 0, 0, FALSE);
2893                                 return;
2894                         } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
2895                                 cam_release_devq(path,
2896                                     /*relsim_flags*/0,
2897                                     /*reduction*/0,
2898                                     /*timeout*/0,
2899                                     /*getcount_only*/0);
2900                         }
2901                 }
2902
2903                 /* Drop freeze taken due to CAM_DEV_QFREEZE */
2904                 cam_release_devq(path, 0, 0, 0, FALSE);
2905
2906                 if ((softc->flags & ADA_FLAG_CAN_LOG)
2907                  && (softc->zone_mode != ADA_ZONE_NONE)) {
2908                         xpt_release_ccb(done_ccb);
2909                         softc->state = ADA_STATE_LOGDIR;
2910                         xpt_schedule(periph, priority);
2911                 } else {
2912                         adaprobedone(periph, done_ccb);
2913                 }
2914                 return;
2915         }
2916         case ADA_CCB_LOGDIR:
2917         {
2918                 int error;
2919
2920                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2921                         error = 0;
2922                         softc->valid_logdir_len = 0;
2923                         bzero(&softc->ata_logdir, sizeof(softc->ata_logdir));
2924                         softc->valid_logdir_len =
2925                                 ataio->dxfer_len - ataio->resid;
2926                         if (softc->valid_logdir_len > 0)
2927                                 bcopy(ataio->data_ptr, &softc->ata_logdir,
2928                                     min(softc->valid_logdir_len,
2929                                         sizeof(softc->ata_logdir)));
2930                         /*
2931                          * Figure out whether the Identify Device log is
2932                          * supported.  The General Purpose log directory
2933                          * has a header, and lists the number of pages
2934                          * available for each GP log identified by the
2935                          * offset into the list.
2936                          */
2937                         if ((softc->valid_logdir_len >=
2938                             ((ATA_IDENTIFY_DATA_LOG + 1) * sizeof(uint16_t)))
2939                          && (le16dec(softc->ata_logdir.header) ==
2940                              ATA_GP_LOG_DIR_VERSION)
2941                          && (le16dec(&softc->ata_logdir.num_pages[
2942                              (ATA_IDENTIFY_DATA_LOG *
2943                              sizeof(uint16_t)) - sizeof(uint16_t)]) > 0)){
2944                                 softc->flags |= ADA_FLAG_CAN_IDLOG;
2945                         } else {
2946                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
2947                         }
2948                 } else {
2949                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
2950                                          SF_RETRY_UA|SF_NO_PRINT);
2951                         if (error == ERESTART)
2952                                 return;
2953                         else if (error != 0) {
2954                                 /*
2955                                  * If we can't get the ATA log directory,
2956                                  * then ATA logs are effectively not
2957                                  * supported even if the bit is set in the
2958                                  * identify data.
2959                                  */
2960                                 softc->flags &= ~(ADA_FLAG_CAN_LOG |
2961                                                   ADA_FLAG_CAN_IDLOG);
2962                                 if ((done_ccb->ccb_h.status &
2963                                      CAM_DEV_QFRZN) != 0) {
2964                                         /* Don't wedge this device's queue */
2965                                         cam_release_devq(done_ccb->ccb_h.path,
2966                                                          /*relsim_flags*/0,
2967                                                          /*reduction*/0,
2968                                                          /*timeout*/0,
2969                                                          /*getcount_only*/0);
2970                                 }
2971                         }
2972
2973
2974                 }
2975
2976                 free(ataio->data_ptr, M_ATADA);
2977
2978                 if ((error == 0)
2979                  && (softc->flags & ADA_FLAG_CAN_IDLOG)) {
2980                         softc->state = ADA_STATE_IDDIR;
2981                         xpt_release_ccb(done_ccb);
2982                         xpt_schedule(periph, priority);
2983                 } else
2984                         adaprobedone(periph, done_ccb);
2985
2986                 return;
2987         }
2988         case ADA_CCB_IDDIR: {
2989                 int error;
2990
2991                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
2992                         off_t entries_offset, max_entries;
2993                         error = 0;
2994
2995                         softc->valid_iddir_len = 0;
2996                         bzero(&softc->ata_iddir, sizeof(softc->ata_iddir));
2997                         softc->flags &= ~(ADA_FLAG_CAN_SUPCAP |
2998                                           ADA_FLAG_CAN_ZONE);
2999                         softc->valid_iddir_len =
3000                                 ataio->dxfer_len - ataio->resid;
3001                         if (softc->valid_iddir_len > 0)
3002                                 bcopy(ataio->data_ptr, &softc->ata_iddir,
3003                                     min(softc->valid_iddir_len,
3004                                         sizeof(softc->ata_iddir)));
3005
3006                         entries_offset =
3007                             __offsetof(struct ata_identify_log_pages,entries);
3008                         max_entries = softc->valid_iddir_len - entries_offset;
3009                         if ((softc->valid_iddir_len > (entries_offset + 1))
3010                          && (le64dec(softc->ata_iddir.header) ==
3011                              ATA_IDLOG_REVISION)
3012                          && (softc->ata_iddir.entry_count > 0)) {
3013                                 int num_entries, i;
3014
3015                                 num_entries = softc->ata_iddir.entry_count;
3016                                 num_entries = min(num_entries,
3017                                    softc->valid_iddir_len - entries_offset);
3018                                 for (i = 0; i < num_entries &&
3019                                      i < max_entries; i++) {
3020                                         if (softc->ata_iddir.entries[i] ==
3021                                             ATA_IDL_SUP_CAP)
3022                                                 softc->flags |=
3023                                                     ADA_FLAG_CAN_SUPCAP;
3024                                         else if (softc->ata_iddir.entries[i]==
3025                                                  ATA_IDL_ZDI)
3026                                                 softc->flags |=
3027                                                     ADA_FLAG_CAN_ZONE;
3028
3029                                         if ((softc->flags &
3030                                              ADA_FLAG_CAN_SUPCAP)
3031                                          && (softc->flags &
3032                                              ADA_FLAG_CAN_ZONE))
3033                                                 break;
3034                                 }
3035                         }
3036                 } else {
3037                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3038                                          SF_RETRY_UA|SF_NO_PRINT);
3039                         if (error == ERESTART)
3040                                 return;
3041                         else if (error != 0) {
3042                                 /*
3043                                  * If we can't get the ATA Identify Data log
3044                                  * directory, then it effectively isn't
3045                                  * supported even if the ATA Log directory
3046                                  * a non-zero number of pages present for
3047                                  * this log.
3048                                  */
3049                                 softc->flags &= ~ADA_FLAG_CAN_IDLOG;
3050                                 if ((done_ccb->ccb_h.status &
3051                                      CAM_DEV_QFRZN) != 0) {
3052                                         /* Don't wedge this device's queue */
3053                                         cam_release_devq(done_ccb->ccb_h.path,
3054                                                          /*relsim_flags*/0,
3055                                                          /*reduction*/0,
3056                                                          /*timeout*/0,
3057                                                          /*getcount_only*/0);
3058                                 }
3059                         }
3060                 }
3061
3062                 free(ataio->data_ptr, M_ATADA);
3063
3064                 if ((error == 0)
3065                  && (softc->flags & ADA_FLAG_CAN_SUPCAP)) {
3066                         softc->state = ADA_STATE_SUP_CAP;
3067                         xpt_release_ccb(done_ccb);
3068                         xpt_schedule(periph, priority);
3069                 } else
3070                         adaprobedone(periph, done_ccb);
3071                 return;
3072         }
3073         case ADA_CCB_SUP_CAP: {
3074                 int error;
3075
3076                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3077                         uint32_t valid_len;
3078                         size_t needed_size;
3079                         struct ata_identify_log_sup_cap *sup_cap;
3080                         error = 0;
3081
3082                         sup_cap = (struct ata_identify_log_sup_cap *)
3083                             ataio->data_ptr;
3084                         valid_len = ataio->dxfer_len - ataio->resid;
3085                         needed_size =
3086                             __offsetof(struct ata_identify_log_sup_cap,
3087                             sup_zac_cap) + 1 + sizeof(sup_cap->sup_zac_cap);
3088                         if (valid_len >= needed_size) {
3089                                 uint64_t zoned, zac_cap;
3090
3091                                 zoned = le64dec(sup_cap->zoned_cap);
3092                                 if (zoned & ATA_ZONED_VALID) {
3093                                         /*
3094                                          * This should have already been
3095                                          * set, because this is also in the
3096                                          * ATA identify data.
3097                                          */
3098                                         if ((zoned & ATA_ZONED_MASK) ==
3099                                             ATA_SUPPORT_ZONE_HOST_AWARE)
3100                                                 softc->zone_mode =
3101                                                     ADA_ZONE_HOST_AWARE;
3102                                         else if ((zoned & ATA_ZONED_MASK) ==
3103                                             ATA_SUPPORT_ZONE_DEV_MANAGED)
3104                                                 softc->zone_mode =
3105                                                     ADA_ZONE_DRIVE_MANAGED;
3106                                 }
3107
3108                                 zac_cap = le64dec(sup_cap->sup_zac_cap);
3109                                 if (zac_cap & ATA_SUP_ZAC_CAP_VALID) {
3110                                         if (zac_cap & ATA_REPORT_ZONES_SUP)
3111                                                 softc->zone_flags |=
3112                                                     ADA_ZONE_FLAG_RZ_SUP;
3113                                         if (zac_cap & ATA_ND_OPEN_ZONE_SUP)
3114                                                 softc->zone_flags |=
3115                                                     ADA_ZONE_FLAG_OPEN_SUP;
3116                                         if (zac_cap & ATA_ND_CLOSE_ZONE_SUP)
3117                                                 softc->zone_flags |=
3118                                                     ADA_ZONE_FLAG_CLOSE_SUP;
3119                                         if (zac_cap & ATA_ND_FINISH_ZONE_SUP)
3120                                                 softc->zone_flags |=
3121                                                     ADA_ZONE_FLAG_FINISH_SUP;
3122                                         if (zac_cap & ATA_ND_RWP_SUP)
3123                                                 softc->zone_flags |=
3124                                                     ADA_ZONE_FLAG_RWP_SUP;
3125                                 } else {
3126                                         /*
3127                                          * This field was introduced in
3128                                          * ACS-4, r08 on April 28th, 2015.
3129                                          * If the drive firmware was written
3130                                          * to an earlier spec, it won't have
3131                                          * the field.  So, assume all
3132                                          * commands are supported.
3133                                          */
3134                                         softc->zone_flags |=
3135                                             ADA_ZONE_FLAG_SUP_MASK;
3136                                 }
3137
3138                         }
3139                 } else {
3140                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3141                                          SF_RETRY_UA|SF_NO_PRINT);
3142                         if (error == ERESTART)
3143                                 return;
3144                         else if (error != 0) {
3145                                 /*
3146                                  * If we can't get the ATA Identify Data
3147                                  * Supported Capabilities page, clear the
3148                                  * flag...
3149                                  */
3150                                 softc->flags &= ~ADA_FLAG_CAN_SUPCAP;
3151                                 /*
3152                                  * And clear zone capabilities.
3153                                  */
3154                                 softc->zone_flags &= ~ADA_ZONE_FLAG_SUP_MASK;
3155                                 if ((done_ccb->ccb_h.status &
3156                                      CAM_DEV_QFRZN) != 0) {
3157                                         /* Don't wedge this device's queue */
3158                                         cam_release_devq(done_ccb->ccb_h.path,
3159                                                          /*relsim_flags*/0,
3160                                                          /*reduction*/0,
3161                                                          /*timeout*/0,
3162                                                          /*getcount_only*/0);
3163                                 }
3164                         }
3165                 }
3166
3167                 free(ataio->data_ptr, M_ATADA);
3168
3169                 if ((error == 0)
3170                  && (softc->flags & ADA_FLAG_CAN_ZONE)) {
3171                         softc->state = ADA_STATE_ZONE;
3172                         xpt_release_ccb(done_ccb);
3173                         xpt_schedule(periph, priority);
3174                 } else
3175                         adaprobedone(periph, done_ccb);
3176                 return;
3177         }
3178         case ADA_CCB_ZONE: {
3179                 int error;
3180
3181                 if ((ataio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
3182                         struct ata_zoned_info_log *zi_log;
3183                         uint32_t valid_len;
3184                         size_t needed_size;
3185
3186                         zi_log = (struct ata_zoned_info_log *)ataio->data_ptr;
3187
3188                         valid_len = ataio->dxfer_len - ataio->resid;
3189                         needed_size = __offsetof(struct ata_zoned_info_log,
3190                             version_info) + 1 + sizeof(zi_log->version_info);
3191                         if (valid_len >= needed_size) {
3192                                 uint64_t tmpvar;
3193
3194                                 tmpvar = le64dec(zi_log->zoned_cap);
3195                                 if (tmpvar & ATA_ZDI_CAP_VALID) {
3196                                         if (tmpvar & ATA_ZDI_CAP_URSWRZ)
3197                                                 softc->zone_flags |=
3198                                                     ADA_ZONE_FLAG_URSWRZ;
3199                                         else
3200                                                 softc->zone_flags &=
3201                                                     ~ADA_ZONE_FLAG_URSWRZ;
3202                                 }
3203                                 tmpvar = le64dec(zi_log->optimal_seq_zones);
3204                                 if (tmpvar & ATA_ZDI_OPT_SEQ_VALID) {
3205                                         softc->zone_flags |=
3206                                             ADA_ZONE_FLAG_OPT_SEQ_SET;
3207                                         softc->optimal_seq_zones = (tmpvar &
3208                                             ATA_ZDI_OPT_SEQ_MASK);
3209                                 } else {
3210                                         softc->zone_flags &=
3211                                             ~ADA_ZONE_FLAG_OPT_SEQ_SET;
3212                                         softc->optimal_seq_zones = 0;
3213                                 }
3214
3215                                 tmpvar =le64dec(zi_log->optimal_nonseq_zones);
3216                                 if (tmpvar & ATA_ZDI_OPT_NS_VALID) {
3217                                         softc->zone_flags |=
3218                                             ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3219                                         softc->optimal_nonseq_zones =
3220                                             (tmpvar & ATA_ZDI_OPT_NS_MASK);
3221                                 } else {
3222                                         softc->zone_flags &=
3223                                             ~ADA_ZONE_FLAG_OPT_NONSEQ_SET;
3224                                         softc->optimal_nonseq_zones = 0;
3225                                 }
3226
3227                                 tmpvar = le64dec(zi_log->max_seq_req_zones);
3228                                 if (tmpvar & ATA_ZDI_MAX_SEQ_VALID) {
3229                                         softc->zone_flags |=
3230                                             ADA_ZONE_FLAG_MAX_SEQ_SET;
3231                                         softc->max_seq_zones =
3232                                             (tmpvar & ATA_ZDI_MAX_SEQ_MASK);
3233                                 } else {
3234                                         softc->zone_flags &=
3235                                             ~ADA_ZONE_FLAG_MAX_SEQ_SET;
3236                                         softc->max_seq_zones = 0;
3237                                 }
3238                         }
3239                 } else {
3240                         error = adaerror(done_ccb, CAM_RETRY_SELTO,
3241                                          SF_RETRY_UA|SF_NO_PRINT);
3242                         if (error == ERESTART)
3243                                 return;
3244                         else if (error != 0) {
3245                                 softc->flags &= ~ADA_FLAG_CAN_ZONE;
3246                                 softc->flags &= ~ADA_ZONE_FLAG_SET_MASK;
3247
3248                                 if ((done_ccb->ccb_h.status &
3249                                      CAM_DEV_QFRZN) != 0) {
3250                                         /* Don't wedge this device's queue */
3251                                         cam_release_devq(done_ccb->ccb_h.path,
3252                                                          /*relsim_flags*/0,
3253                                                          /*reduction*/0,
3254                                                          /*timeout*/0,
3255                                                          /*getcount_only*/0);
3256                                 }
3257                         }
3258
3259                 }
3260                 free(ataio->data_ptr, M_ATADA);
3261
3262                 adaprobedone(periph, done_ccb);
3263                 return;
3264         }
3265         case ADA_CCB_DUMP:
3266                 /* No-op.  We're polling */
3267                 return;
3268         default:
3269                 break;
3270         }
3271         xpt_release_ccb(done_ccb);
3272 }
3273
3274 static int
3275 adaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
3276 {
3277 #ifdef CAM_IO_STATS
3278         struct ada_softc *softc;
3279         struct cam_periph *periph;
3280
3281         periph = xpt_path_periph(ccb->ccb_h.path);
3282         softc = (struct ada_softc *)periph->softc;
3283
3284         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
3285         case CAM_CMD_TIMEOUT:
3286                 softc->timeouts++;
3287                 break;
3288         case CAM_REQ_ABORTED:
3289         case CAM_REQ_CMP_ERR:
3290         case CAM_REQ_TERMIO:
3291         case CAM_UNREC_HBA_ERROR:
3292         case CAM_DATA_RUN_ERR:
3293         case CAM_ATA_STATUS_ERROR:
3294                 softc->errors++;
3295                 break;
3296         default:
3297                 break;
3298         }
3299 #endif
3300
3301         return(cam_periph_error(ccb, cam_flags, sense_flags, NULL));
3302 }
3303
3304 static void
3305 adagetparams(struct cam_periph *periph, struct ccb_getdev *cgd)
3306 {
3307         struct ada_softc *softc = (struct ada_softc *)periph->softc;
3308         struct disk_params *dp = &softc->params;
3309         u_int64_t lbasize48;
3310         u_int32_t lbasize;
3311
3312         dp->secsize = ata_logical_sector_size(&cgd->ident_data);
3313         if ((cgd->ident_data.atavalid & ATA_FLAG_54_58) &&
3314                 cgd->ident_data.current_heads && cgd->ident_data.current_sectors) {
3315                 dp->heads = cgd->ident_data.current_heads;
3316                 dp->secs_per_track = cgd->ident_data.current_sectors;
3317                 dp->cylinders = cgd->ident_data.cylinders;
3318                 dp->sectors = (u_int32_t)cgd->ident_data.current_size_1 |
3319                           ((u_int32_t)cgd->ident_data.current_size_2 << 16);
3320         } else {
3321                 dp->heads = cgd->ident_data.heads;
3322                 dp->secs_per_track = cgd->ident_data.sectors;
3323                 dp->cylinders = cgd->ident_data.cylinders;
3324                 dp->sectors = cgd->ident_data.cylinders * dp->heads * dp->secs_per_track;
3325         }
3326         lbasize = (u_int32_t)cgd->ident_data.lba_size_1 |
3327                   ((u_int32_t)cgd->ident_data.lba_size_2 << 16);
3328
3329         /* use the 28bit LBA size if valid or bigger than the CHS mapping */
3330         if (cgd->ident_data.cylinders == 16383 || dp->sectors < lbasize)
3331                 dp->sectors = lbasize;
3332
3333         /* use the 48bit LBA size if valid */
3334         lbasize48 = ((u_int64_t)cgd->ident_data.lba_size48_1) |
3335                     ((u_int64_t)cgd->ident_data.lba_size48_2 << 16) |
3336                     ((u_int64_t)cgd->ident_data.lba_size48_3 << 32) |
3337                     ((u_int64_t)cgd->ident_data.lba_size48_4 << 48);
3338         if ((cgd->ident_data.support.command2 & ATA_SUPPORT_ADDRESS48) &&
3339             lbasize48 > ATA_MAX_28BIT_LBA)
3340                 dp->sectors = lbasize48;
3341 }
3342
3343 static void
3344 adasendorderedtag(void *arg)
3345 {
3346         struct ada_softc *softc = arg;
3347
3348         if (ada_send_ordered) {
3349                 if (softc->outstanding_cmds > 0) {
3350                         if ((softc->flags & ADA_FLAG_WAS_OTAG) == 0)
3351                                 softc->flags |= ADA_FLAG_NEED_OTAG;
3352                         softc->flags &= ~ADA_FLAG_WAS_OTAG;
3353                 }
3354         }
3355         /* Queue us up again */
3356         callout_reset(&softc->sendordered_c,
3357             (ada_default_timeout * hz) / ADA_ORDEREDTAG_INTERVAL,
3358             adasendorderedtag, softc);
3359 }
3360
3361 /*
3362  * Step through all ADA peripheral drivers, and if the device is still open,
3363  * sync the disk cache to physical media.
3364  */
3365 static void
3366 adaflush(void)
3367 {
3368         struct cam_periph *periph;
3369         struct ada_softc *softc;
3370         union ccb *ccb;
3371         int error;
3372
3373         CAM_PERIPH_FOREACH(periph, &adadriver) {
3374                 softc = (struct ada_softc *)periph->softc;
3375                 if (SCHEDULER_STOPPED()) {
3376                         /* If we paniced with the lock held, do not recurse. */
3377                         if (!cam_periph_owned(periph) &&
3378                             (softc->flags & ADA_FLAG_OPEN)) {
3379                                 adadump(softc->disk, NULL, 0, 0, 0);
3380                         }
3381                         continue;
3382                 }
3383                 cam_periph_lock(periph);
3384                 /*
3385                  * We only sync the cache if the drive is still open, and
3386                  * if the drive is capable of it..
3387                  */
3388                 if (((softc->flags & ADA_FLAG_OPEN) == 0) ||
3389                     (softc->flags & ADA_FLAG_CAN_FLUSHCACHE) == 0) {
3390                         cam_periph_unlock(periph);
3391                         continue;
3392                 }
3393
3394                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3395                 cam_fill_ataio(&ccb->ataio,
3396                                     0,
3397                                     adadone,
3398                                     CAM_DIR_NONE,
3399                                     0,
3400                                     NULL,
3401                                     0,
3402                                     ada_default_timeout*1000);
3403                 if (softc->flags & ADA_FLAG_CAN_48BIT)
3404                         ata_48bit_cmd(&ccb->ataio, ATA_FLUSHCACHE48, 0, 0, 0);
3405                 else
3406                         ata_28bit_cmd(&ccb->ataio, ATA_FLUSHCACHE, 0, 0, 0);
3407
3408                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3409                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3410                     softc->disk->d_devstat);
3411                 if (error != 0)
3412                         xpt_print(periph->path, "Synchronize cache failed\n");
3413                 xpt_release_ccb(ccb);
3414                 cam_periph_unlock(periph);
3415         }
3416 }
3417
3418 static void
3419 adaspindown(uint8_t cmd, int flags)
3420 {
3421         struct cam_periph *periph;
3422         struct ada_softc *softc;
3423         union ccb *ccb;
3424         int error;
3425
3426         CAM_PERIPH_FOREACH(periph, &adadriver) {
3427                 /* If we paniced with lock held - not recurse here. */
3428                 if (cam_periph_owned(periph))
3429                         continue;
3430                 cam_periph_lock(periph);
3431                 softc = (struct ada_softc *)periph->softc;
3432                 /*
3433                  * We only spin-down the drive if it is capable of it..
3434                  */
3435                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3436                         cam_periph_unlock(periph);
3437                         continue;
3438                 }
3439
3440                 if (bootverbose)
3441                         xpt_print(periph->path, "spin-down\n");
3442
3443                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
3444                 cam_fill_ataio(&ccb->ataio,
3445                                     0,
3446                                     adadone,
3447                                     CAM_DIR_NONE | flags,
3448                                     0,
3449                                     NULL,
3450                                     0,
3451                                     ada_default_timeout*1000);
3452                 ata_28bit_cmd(&ccb->ataio, cmd, 0, 0, 0);
3453
3454                 error = cam_periph_runccb(ccb, adaerror, /*cam_flags*/0,
3455                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
3456                     softc->disk->d_devstat);
3457                 if (error != 0)
3458                         xpt_print(periph->path, "Spin-down disk failed\n");
3459                 xpt_release_ccb(ccb);
3460                 cam_periph_unlock(periph);
3461         }
3462 }
3463
3464 static void
3465 adashutdown(void *arg, int howto)
3466 {
3467
3468         adaflush();
3469         if (ada_spindown_shutdown != 0 &&
3470             (howto & (RB_HALT | RB_POWEROFF)) != 0)
3471                 adaspindown(ATA_STANDBY_IMMEDIATE, 0);
3472 }
3473
3474 static void
3475 adasuspend(void *arg)
3476 {
3477
3478         adaflush();
3479         if (ada_spindown_suspend != 0)
3480                 adaspindown(ATA_SLEEP, CAM_DEV_QFREEZE);
3481 }
3482
3483 static void
3484 adaresume(void *arg)
3485 {
3486         struct cam_periph *periph;
3487         struct ada_softc *softc;
3488
3489         if (ada_spindown_suspend == 0)
3490                 return;
3491
3492         CAM_PERIPH_FOREACH(periph, &adadriver) {
3493                 cam_periph_lock(periph);
3494                 softc = (struct ada_softc *)periph->softc;
3495                 /*
3496                  * We only spin-down the drive if it is capable of it..
3497                  */
3498                 if ((softc->flags & ADA_FLAG_CAN_POWERMGT) == 0) {
3499                         cam_periph_unlock(periph);
3500                         continue;
3501                 }
3502
3503                 if (bootverbose)
3504                         xpt_print(periph->path, "resume\n");
3505
3506                 /*
3507                  * Drop freeze taken due to CAM_DEV_QFREEZE flag set on
3508                  * sleep request.
3509                  */
3510                 cam_release_devq(periph->path,
3511                          /*relsim_flags*/0,
3512                          /*openings*/0,
3513                          /*timeout*/0,
3514                          /*getcount_only*/0);
3515
3516                 cam_periph_unlock(periph);
3517         }
3518 }
3519
3520 #endif /* _KERNEL */