sys/dev/nvme/nvme.h

   1 /*-
   2  * Copyright (C) 2012-2013 Intel Corporation
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #ifndef __NVME_H__
  30 #define __NVME_H__
  31
  32 #ifdef _KERNEL
  33 #include <sys/types.h>
  34 #endif
  35
  36 #include <sys/param.h>
  37
  38 #define NVME_PASSTHROUGH_CMD            _IOWR('n', 0, struct nvme_pt_command)
  39 #define NVME_RESET_CONTROLLER           _IO('n', 1)
  40
  41 #define NVME_IO_TEST                    _IOWR('n', 100, struct nvme_io_test)
  42 #define NVME_BIO_TEST                   _IOWR('n', 101, struct nvme_io_test)
  43
  44 /*
  45  * Macros to deal with NVME revisions, as defined VS register
  46  */
  47 #define NVME_REV(x, y)                  (((x) << 16) | ((y) << 8))
  48 #define NVME_MAJOR(r)                   (((r) >> 16) & 0xffff)
  49 #define NVME_MINOR(r)                   (((r) >> 8) & 0xff)
  50
  51 /*
  52  * Use to mark a command to apply to all namespaces, or to retrieve global
  53  *  log pages.
  54  */
  55 #define NVME_GLOBAL_NAMESPACE_TAG       ((uint32_t)0xFFFFFFFF)
  56
  57 /* Cap nvme to 1MB transfers driver explodes with larger sizes */
  58 #define NVME_MAX_XFER_SIZE              (MAXPHYS < (1<<20) ? MAXPHYS : (1<<20))
  59
  60 union cap_lo_register {
  61         uint32_t        raw;
  62         struct {
  63                 /** maximum queue entries supported */
  64                 uint32_t mqes           : 16;
  65
  66                 /** contiguous queues required */
  67                 uint32_t cqr            : 1;
  68
  69                 /** arbitration mechanism supported */
  70                 uint32_t ams            : 2;
  71
  72                 uint32_t reserved1      : 5;
  73
  74                 /** timeout */
  75                 uint32_t to             : 8;
  76         } bits __packed;
  77 } __packed;
  78
  79 _Static_assert(sizeof(union cap_lo_register) == 4, "bad size for cap_lo_register");
  80
  81 union cap_hi_register {
  82         uint32_t        raw;
  83         struct {
  84                 /** doorbell stride */
  85                 uint32_t dstrd          : 4;
  86
  87                 uint32_t reserved3      : 1;
  88
  89                 /** command sets supported */
  90                 uint32_t css_nvm        : 1;
  91
  92                 uint32_t css_reserved   : 3;
  93                 uint32_t reserved2      : 7;
  94
  95                 /** memory page size minimum */
  96                 uint32_t mpsmin         : 4;
  97
  98                 /** memory page size maximum */
  99                 uint32_t mpsmax         : 4;
 100
 101                 uint32_t reserved1      : 8;
 102         } bits __packed;
 103 } __packed;
 104
 105 _Static_assert(sizeof(union cap_hi_register) == 4, "bad size of cap_hi_register");
 106
 107 union cc_register {
 108         uint32_t        raw;
 109         struct {
 110                 /** enable */
 111                 uint32_t en             : 1;
 112
 113                 uint32_t reserved1      : 3;
 114
 115                 /** i/o command set selected */
 116                 uint32_t css            : 3;
 117
 118                 /** memory page size */
 119                 uint32_t mps            : 4;
 120
 121                 /** arbitration mechanism selected */
 122                 uint32_t ams            : 3;
 123
 124                 /** shutdown notification */
 125                 uint32_t shn            : 2;
 126
 127                 /** i/o submission queue entry size */
 128                 uint32_t iosqes         : 4;
 129
 130                 /** i/o completion queue entry size */
 131                 uint32_t iocqes         : 4;
 132
 133                 uint32_t reserved2      : 8;
 134         } bits __packed;
 135 } __packed;
 136
 137 _Static_assert(sizeof(union cc_register) == 4, "bad size for cc_register");
 138
 139 enum shn_value {
 140         NVME_SHN_NORMAL         = 0x1,
 141         NVME_SHN_ABRUPT         = 0x2,
 142 };
 143
 144 union csts_register {
 145         uint32_t        raw;
 146         struct {
 147                 /** ready */
 148                 uint32_t rdy            : 1;
 149
 150                 /** controller fatal status */
 151                 uint32_t cfs            : 1;
 152
 153                 /** shutdown status */
 154                 uint32_t shst           : 2;
 155
 156                 uint32_t reserved1      : 28;
 157         } bits __packed;
 158 } __packed;
 159
 160 _Static_assert(sizeof(union csts_register) == 4, "bad size for csts_register");
 161
 162 enum shst_value {
 163         NVME_SHST_NORMAL        = 0x0,
 164         NVME_SHST_OCCURRING     = 0x1,
 165         NVME_SHST_COMPLETE      = 0x2,
 166 };
 167
 168 union aqa_register {
 169         uint32_t        raw;
 170         struct {
 171                 /** admin submission queue size */
 172                 uint32_t asqs           : 12;
 173
 174                 uint32_t reserved1      : 4;
 175
 176                 /** admin completion queue size */
 177                 uint32_t acqs           : 12;
 178
 179                 uint32_t reserved2      : 4;
 180         } bits __packed;
 181 } __packed;
 182
 183 _Static_assert(sizeof(union aqa_register) == 4, "bad size for aqa_resgister");
 184
 185 struct nvme_registers
 186 {
 187         /** controller capabilities */
 188         union cap_lo_register   cap_lo;
 189         union cap_hi_register   cap_hi;
 190
 191         uint32_t                vs;     /* version */
 192         uint32_t                intms;  /* interrupt mask set */
 193         uint32_t                intmc;  /* interrupt mask clear */
 194
 195         /** controller configuration */
 196         union cc_register       cc;
 197
 198         uint32_t                reserved1;
 199
 200         /** controller status */
 201         union csts_register     csts;
 202
 203         uint32_t                reserved2;
 204
 205         /** admin queue attributes */
 206         union aqa_register      aqa;
 207
 208         uint64_t                asq;    /* admin submission queue base addr */
 209         uint64_t                acq;    /* admin completion queue base addr */
 210         uint32_t                reserved3[0x3f2];
 211
 212         struct {
 213             uint32_t            sq_tdbl; /* submission queue tail doorbell */
 214             uint32_t            cq_hdbl; /* completion queue head doorbell */
 215         } doorbell[1] __packed;
 216 } __packed;
 217
 218 _Static_assert(sizeof(struct nvme_registers) == 0x1008, "bad size for nvme_registers");
 219
 220 struct nvme_command
 221 {
 222         /* dword 0 */
 223         uint16_t opc    :  8;   /* opcode */
 224         uint16_t fuse   :  2;   /* fused operation */
 225         uint16_t rsvd1  :  6;
 226         uint16_t cid;           /* command identifier */
 227
 228         /* dword 1 */
 229         uint32_t nsid;          /* namespace identifier */
 230
 231         /* dword 2-3 */
 232         uint32_t rsvd2;
 233         uint32_t rsvd3;
 234
 235         /* dword 4-5 */
 236         uint64_t mptr;          /* metadata pointer */
 237
 238         /* dword 6-7 */
 239         uint64_t prp1;          /* prp entry 1 */
 240
 241         /* dword 8-9 */
 242         uint64_t prp2;          /* prp entry 2 */
 243
 244         /* dword 10-15 */
 245         uint32_t cdw10;         /* command-specific */
 246         uint32_t cdw11;         /* command-specific */
 247         uint32_t cdw12;         /* command-specific */
 248         uint32_t cdw13;         /* command-specific */
 249         uint32_t cdw14;         /* command-specific */
 250         uint32_t cdw15;         /* command-specific */
 251 } __packed;
 252
 253 _Static_assert(sizeof(struct nvme_command) == 16 * 4, "bad size for nvme_command");
 254
 255 struct nvme_status {
 256
 257         uint16_t p      :  1;   /* phase tag */
 258         uint16_t sc     :  8;   /* status code */
 259         uint16_t sct    :  3;   /* status code type */
 260         uint16_t rsvd2  :  2;
 261         uint16_t m      :  1;   /* more */
 262         uint16_t dnr    :  1;   /* do not retry */
 263 } __packed;
 264
 265 _Static_assert(sizeof(struct nvme_status) == 2, "bad size for nvme_status");
 266
 267 struct nvme_completion {
 268
 269         /* dword 0 */
 270         uint32_t                cdw0;   /* command-specific */
 271
 272         /* dword 1 */
 273         uint32_t                rsvd1;
 274
 275         /* dword 2 */
 276         uint16_t                sqhd;   /* submission queue head pointer */
 277         uint16_t                sqid;   /* submission queue identifier */
 278
 279         /* dword 3 */
 280         uint16_t                cid;    /* command identifier */
 281         struct nvme_status      status;
 282 } __packed;
 283
 284 _Static_assert(sizeof(struct nvme_completion) == 4 * 4, "bad size for nvme_completion");
 285
 286 struct nvme_dsm_range {
 287
 288         uint32_t attributes;
 289         uint32_t length;
 290         uint64_t starting_lba;
 291 } __packed;
 292
 293 _Static_assert(sizeof(struct nvme_dsm_range) == 16, "bad size for nvme_dsm_ranage");
 294
 295 /* status code types */
 296 enum nvme_status_code_type {
 297         NVME_SCT_GENERIC                = 0x0,
 298         NVME_SCT_COMMAND_SPECIFIC       = 0x1,
 299         NVME_SCT_MEDIA_ERROR            = 0x2,
 300         /* 0x3-0x6 - reserved */
 301         NVME_SCT_VENDOR_SPECIFIC        = 0x7,
 302 };
 303
 304 /* generic command status codes */
 305 enum nvme_generic_command_status_code {
 306         NVME_SC_SUCCESS                         = 0x00,
 307         NVME_SC_INVALID_OPCODE                  = 0x01,
 308         NVME_SC_INVALID_FIELD                   = 0x02,
 309         NVME_SC_COMMAND_ID_CONFLICT             = 0x03,
 310         NVME_SC_DATA_TRANSFER_ERROR             = 0x04,
 311         NVME_SC_ABORTED_POWER_LOSS              = 0x05,
 312         NVME_SC_INTERNAL_DEVICE_ERROR           = 0x06,
 313         NVME_SC_ABORTED_BY_REQUEST              = 0x07,
 314         NVME_SC_ABORTED_SQ_DELETION             = 0x08,
 315         NVME_SC_ABORTED_FAILED_FUSED            = 0x09,
 316         NVME_SC_ABORTED_MISSING_FUSED           = 0x0a,
 317         NVME_SC_INVALID_NAMESPACE_OR_FORMAT     = 0x0b,
 318         NVME_SC_COMMAND_SEQUENCE_ERROR          = 0x0c,
 319         NVME_SC_INVALID_SGL_SEGMENT_DESCR       = 0x0d,
 320         NVME_SC_INVALID_NUMBER_OF_SGL_DESCR     = 0x0e,
 321         NVME_SC_DATA_SGL_LENGTH_INVALID         = 0x0f,
 322         NVME_SC_METADATA_SGL_LENGTH_INVALID     = 0x10,
 323         NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID     = 0x11,
 324         NVME_SC_INVALID_USE_OF_CMB              = 0x12,
 325         NVME_SC_PRP_OFFET_INVALID               = 0x13,
 326         NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED      = 0x14,
 327         NVME_SC_OPERATION_DENIED                = 0x15,
 328         NVME_SC_SGL_OFFSET_INVALID              = 0x16,
 329         /* 0x17 - reserved */
 330         NVME_SC_HOST_ID_INCONSISTENT_FORMAT     = 0x18,
 331         NVME_SC_KEEP_ALIVE_TIMEOUT_EXPIRED      = 0x19,
 332         NVME_SC_KEEP_ALIVE_TIMEOUT_INVALID      = 0x1a,
 333         NVME_SC_ABORTED_DUE_TO_PREEMPT          = 0x1b,
 334         NVME_SC_SANITIZE_FAILED                 = 0x1c,
 335         NVME_SC_SANITIZE_IN_PROGRESS            = 0x1d,
 336         NVME_SC_SGL_DATA_BLOCK_GRAN_INVALID     = 0x1e,
 337         NVME_SC_NOT_SUPPORTED_IN_CMB            = 0x1f,
 338
 339         NVME_SC_LBA_OUT_OF_RANGE                = 0x80,
 340         NVME_SC_CAPACITY_EXCEEDED               = 0x81,
 341         NVME_SC_NAMESPACE_NOT_READY             = 0x82,
 342         NVME_SC_RESERVATION_CONFLICT            = 0x83,
 343         NVME_SC_FORMAT_IN_PROGRESS              = 0x84,
 344 };
 345
 346 /* command specific status codes */
 347 enum nvme_command_specific_status_code {
 348         NVME_SC_COMPLETION_QUEUE_INVALID        = 0x00,
 349         NVME_SC_INVALID_QUEUE_IDENTIFIER        = 0x01,
 350         NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED     = 0x02,
 351         NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED    = 0x03,
 352         /* 0x04 - reserved */
 353         NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05,
 354         NVME_SC_INVALID_FIRMWARE_SLOT           = 0x06,
 355         NVME_SC_INVALID_FIRMWARE_IMAGE          = 0x07,
 356         NVME_SC_INVALID_INTERRUPT_VECTOR        = 0x08,
 357         NVME_SC_INVALID_LOG_PAGE                = 0x09,
 358         NVME_SC_INVALID_FORMAT                  = 0x0a,
 359         NVME_SC_FIRMWARE_REQUIRES_RESET         = 0x0b,
 360         NVME_SC_INVALID_QUEUE_DELETION          = 0x0c,
 361         NVME_SC_FEATURE_NOT_SAVEABLE            = 0x0d,
 362         NVME_SC_FEATURE_NOT_CHANGEABLE          = 0x0e,
 363         NVME_SC_FEATURE_NOT_NS_SPECIFIC         = 0x0f,
 364         NVME_SC_FW_ACT_REQUIRES_NVMS_RESET      = 0x10,
 365         NVME_SC_FW_ACT_REQUIRES_RESET           = 0x11,
 366         NVME_SC_FW_ACT_REQUIRES_TIME            = 0x12,
 367         NVME_SC_FW_ACT_PROHIBITED               = 0x13,
 368         NVME_SC_OVERLAPPING_RANGE               = 0x14,
 369         NVME_SC_NS_INSUFFICIENT_CAPACITY        = 0x15,
 370         NVME_SC_NS_ID_UNAVAILABLE               = 0x16,
 371         /* 0x17 - reserved */
 372         NVME_SC_NS_ALREADY_ATTACHED             = 0x18,
 373         NVME_SC_NS_IS_PRIVATE                   = 0x19,
 374         NVME_SC_NS_NOT_ATTACHED                 = 0x1a,
 375         NVME_SC_THIN_PROV_NOT_SUPPORTED         = 0x1b,
 376         NVME_SC_CTRLR_LIST_INVALID              = 0x1c,
 377         NVME_SC_SELT_TEST_IN_PROGRESS           = 0x1d,
 378         NVME_SC_BOOT_PART_WRITE_PROHIB          = 0x1e,
 379         NVME_SC_INVALID_CTRLR_ID                = 0x1f,
 380         NVME_SC_INVALID_SEC_CTRLR_STATE         = 0x20,
 381         NVME_SC_INVALID_NUM_OF_CTRLR_RESRC      = 0x21,
 382         NVME_SC_INVALID_RESOURCE_ID             = 0x22,
 383
 384         NVME_SC_CONFLICTING_ATTRIBUTES          = 0x80,
 385         NVME_SC_INVALID_PROTECTION_INFO         = 0x81,
 386         NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE      = 0x82,
 387 };
 388
 389 /* media error status codes */
 390 enum nvme_media_error_status_code {
 391         NVME_SC_WRITE_FAULTS                    = 0x80,
 392         NVME_SC_UNRECOVERED_READ_ERROR          = 0x81,
 393         NVME_SC_GUARD_CHECK_ERROR               = 0x82,
 394         NVME_SC_APPLICATION_TAG_CHECK_ERROR     = 0x83,
 395         NVME_SC_REFERENCE_TAG_CHECK_ERROR       = 0x84,
 396         NVME_SC_COMPARE_FAILURE                 = 0x85,
 397         NVME_SC_ACCESS_DENIED                   = 0x86,
 398         NVME_SC_DEALLOCATED_OR_UNWRITTEN        = 0x87,
 399 };
 400
 401 /* admin opcodes */
 402 enum nvme_admin_opcode {
 403         NVME_OPC_DELETE_IO_SQ                   = 0x00,
 404         NVME_OPC_CREATE_IO_SQ                   = 0x01,
 405         NVME_OPC_GET_LOG_PAGE                   = 0x02,
 406         /* 0x03 - reserved */
 407         NVME_OPC_DELETE_IO_CQ                   = 0x04,
 408         NVME_OPC_CREATE_IO_CQ                   = 0x05,
 409         NVME_OPC_IDENTIFY                       = 0x06,
 410         /* 0x07 - reserved */
 411         NVME_OPC_ABORT                          = 0x08,
 412         NVME_OPC_SET_FEATURES                   = 0x09,
 413         NVME_OPC_GET_FEATURES                   = 0x0a,
 414         /* 0x0b - reserved */
 415         NVME_OPC_ASYNC_EVENT_REQUEST            = 0x0c,
 416         NVME_OPC_NAMESPACE_MANAGEMENT           = 0x0d,
 417         /* 0x0e-0x0f - reserved */
 418         NVME_OPC_FIRMWARE_ACTIVATE              = 0x10,
 419         NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD        = 0x11,
 420         NVME_OPC_DEVICE_SELF_TEST               = 0x14,
 421         NVME_OPC_NAMESPACE_ATTACHMENT           = 0x15,
 422         NVME_OPC_KEEP_ALIVE                     = 0x18,
 423         NVME_OPC_DIRECTIVE_SEND                 = 0x19,
 424         NVME_OPC_DIRECTIVE_RECEIVE              = 0x1a,
 425         NVME_OPC_VIRTUALIZATION_MANAGEMENT      = 0x1c,
 426         NVME_OPC_NVME_MI_SEND                   = 0x1d,
 427         NVME_OPC_NVME_MI_RECEIVE                = 0x1e,
 428         NVME_OPC_DOORBELL_BUFFER_CONFIG         = 0x7c,
 429
 430         NVME_OPC_FORMAT_NVM                     = 0x80,
 431         NVME_OPC_SECURITY_SEND                  = 0x81,
 432         NVME_OPC_SECURITY_RECEIVE               = 0x82,
 433         NVME_OPC_SANITIZE                       = 0x84,
 434 };
 435
 436 /* nvme nvm opcodes */
 437 enum nvme_nvm_opcode {
 438         NVME_OPC_FLUSH                          = 0x00,
 439         NVME_OPC_WRITE                          = 0x01,
 440         NVME_OPC_READ                           = 0x02,
 441         /* 0x03 - reserved */
 442         NVME_OPC_WRITE_UNCORRECTABLE            = 0x04,
 443         NVME_OPC_COMPARE                        = 0x05,
 444         /* 0x06 - reserved */
 445         NVME_OPC_WRITE_ZEROES                   = 0x08,
 446         /* 0x07 - reserved */
 447         NVME_OPC_DATASET_MANAGEMENT             = 0x09,
 448         /* 0x0a-0x0c - reserved */
 449         NVME_OPC_RESERVATION_REGISTER           = 0x0d,
 450         NVME_OPC_RESERVATION_REPORT             = 0x0e,
 451         /* 0x0f-0x10 - reserved */
 452         NVME_OPC_RESERVATION_ACQUIRE            = 0x11,
 453         /* 0x12-0x14 - reserved */
 454         NVME_OPC_RESERVATION_RELEASE            = 0x15,
 455 };
 456
 457 enum nvme_feature {
 458         /* 0x00 - reserved */
 459         NVME_FEAT_ARBITRATION                   = 0x01,
 460         NVME_FEAT_POWER_MANAGEMENT              = 0x02,
 461         NVME_FEAT_LBA_RANGE_TYPE                = 0x03,
 462         NVME_FEAT_TEMPERATURE_THRESHOLD         = 0x04,
 463         NVME_FEAT_ERROR_RECOVERY                = 0x05,
 464         NVME_FEAT_VOLATILE_WRITE_CACHE          = 0x06,
 465         NVME_FEAT_NUMBER_OF_QUEUES              = 0x07,
 466         NVME_FEAT_INTERRUPT_COALESCING          = 0x08,
 467         NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09,
 468         NVME_FEAT_WRITE_ATOMICITY               = 0x0A,
 469         NVME_FEAT_ASYNC_EVENT_CONFIGURATION     = 0x0B,
 470         NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION = 0x0C,
 471         NVME_FEAT_HOST_MEMORY_BUFFER            = 0x0D,
 472         NVME_FEAT_TIMESTAMP                     = 0x0E,
 473         NVME_FEAT_KEEP_ALIVE_TIMER              = 0x0F,
 474         NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT  = 0x10,
 475         NVME_FEAT_NON_OP_POWER_STATE_CONFIG     = 0x11,
 476         /* 0x12-0x77 - reserved */
 477         /* 0x78-0x7f - NVMe Management Interface */
 478         NVME_FEAT_SOFTWARE_PROGRESS_MARKER      = 0x80,
 479         /* 0x81-0xBF - command set specific (reserved) */
 480         /* 0xC0-0xFF - vendor specific */
 481 };
 482
 483 enum nvme_dsm_attribute {
 484         NVME_DSM_ATTR_INTEGRAL_READ             = 0x1,
 485         NVME_DSM_ATTR_INTEGRAL_WRITE            = 0x2,
 486         NVME_DSM_ATTR_DEALLOCATE                = 0x4,
 487 };
 488
 489 enum nvme_activate_action {
 490         NVME_AA_REPLACE_NO_ACTIVATE             = 0x0,
 491         NVME_AA_REPLACE_ACTIVATE                = 0x1,
 492         NVME_AA_ACTIVATE                        = 0x2,
 493 };
 494
 495 struct nvme_power_state {
 496         /** Maximum Power */
 497         uint16_t        mp;                     /* Maximum Power */
 498         uint8_t         ps_rsvd1;
 499         uint8_t         mps      : 1;           /* Max Power Scale */
 500         uint8_t         nops     : 1;           /* Non-Operational State */
 501         uint8_t         ps_rsvd2 : 6;
 502         uint32_t        enlat;                  /* Entry Latency */
 503         uint32_t        exlat;                  /* Exit Latency */
 504         uint8_t         rrt      : 5;           /* Relative Read Throughput */
 505         uint8_t         ps_rsvd3 : 3;
 506         uint8_t         rrl      : 5;           /* Relative Read Latency */
 507         uint8_t         ps_rsvd4 : 3;
 508         uint8_t         rwt      : 5;           /* Relative Write Throughput */
 509         uint8_t         ps_rsvd5 : 3;
 510         uint8_t         rwl      : 5;           /* Relative Write Latency */
 511         uint8_t         ps_rsvd6 : 3;
 512         uint16_t        idlp;                   /* Idle Power */
 513         uint8_t         ps_rsvd7 : 6;
 514         uint8_t         ips      : 2;           /* Idle Power Scale */
 515         uint8_t         ps_rsvd8;
 516         uint16_t        actp;                   /* Active Power */
 517         uint8_t         apw      : 3;           /* Active Power Workload */
 518         uint8_t         ps_rsvd9 : 3;
 519         uint8_t         aps      : 2;           /* Active Power Scale */
 520         uint8_t         ps_rsvd10[9];
 521 } __packed;
 522
 523 _Static_assert(sizeof(struct nvme_power_state) == 32, "bad size for nvme_power_state");
 524
 525 #define NVME_SERIAL_NUMBER_LENGTH       20
 526 #define NVME_MODEL_NUMBER_LENGTH        40
 527 #define NVME_FIRMWARE_REVISION_LENGTH   8
 528
 529 struct nvme_controller_data {
 530
 531         /* bytes 0-255: controller capabilities and features */
 532
 533         /** pci vendor id */
 534         uint16_t                vid;
 535
 536         /** pci subsystem vendor id */
 537         uint16_t                ssvid;
 538
 539         /** serial number */
 540         uint8_t                 sn[NVME_SERIAL_NUMBER_LENGTH];
 541
 542         /** model number */
 543         uint8_t                 mn[NVME_MODEL_NUMBER_LENGTH];
 544
 545         /** firmware revision */
 546         uint8_t                 fr[NVME_FIRMWARE_REVISION_LENGTH];
 547
 548         /** recommended arbitration burst */
 549         uint8_t                 rab;
 550
 551         /** ieee oui identifier */
 552         uint8_t                 ieee[3];
 553
 554         /** multi-interface capabilities */
 555         uint8_t                 mic;
 556
 557         /** maximum data transfer size */
 558         uint8_t                 mdts;
 559
 560         /** Controller ID */
 561         uint16_t                ctrlr_id;
 562
 563         /** Version */
 564         uint32_t                ver;
 565
 566         /** RTD3 Resume Latency */
 567         uint32_t                rtd3r;
 568
 569         /** RTD3 Enter Latency */
 570         uint32_t                rtd3e;
 571
 572         /** Optional Asynchronous Events Supported */
 573         uint32_t                oaes;   /* bitfield really */
 574
 575         /** Controller Attributes */
 576         uint32_t                ctratt; /* bitfield really */
 577
 578         uint8_t                 reserved1[12];
 579
 580         /** FRU Globally Unique Identifier */
 581         uint8_t                 fguid[16];
 582
 583         uint8_t                 reserved2[128];
 584
 585         /* bytes 256-511: admin command set attributes */
 586
 587         /** optional admin command support */
 588         struct {
 589                 /* supports security send/receive commands */
 590                 uint16_t        security  : 1;
 591
 592                 /* supports format nvm command */
 593                 uint16_t        format    : 1;
 594
 595                 /* supports firmware activate/download commands */
 596                 uint16_t        firmware  : 1;
 597
 598                 /* supports namespace management commands */
 599                 uint16_t        nsmgmt    : 1;
 600
 601                 uint16_t        oacs_rsvd : 12;
 602         } __packed oacs;
 603
 604         /** abort command limit */
 605         uint8_t                 acl;
 606
 607         /** asynchronous event request limit */
 608         uint8_t                 aerl;
 609
 610         /** firmware updates */
 611         struct {
 612                 /* first slot is read-only */
 613                 uint8_t         slot1_ro  : 1;
 614
 615                 /* number of firmware slots */
 616                 uint8_t         num_slots : 3;
 617
 618                 uint8_t         frmw_rsvd : 4;
 619         } __packed frmw;
 620
 621         /** log page attributes */
 622         struct {
 623                 /* per namespace smart/health log page */
 624                 uint8_t         ns_smart : 1;
 625
 626                 uint8_t         lpa_rsvd : 7;
 627         } __packed lpa;
 628
 629         /** error log page entries */
 630         uint8_t                 elpe;
 631
 632         /** number of power states supported */
 633         uint8_t                 npss;
 634
 635         /** admin vendor specific command configuration */
 636         struct {
 637                 /* admin vendor specific commands use spec format */
 638                 uint8_t         spec_format : 1;
 639
 640                 uint8_t         avscc_rsvd  : 7;
 641         } __packed avscc;
 642
 643         /** Autonomous Power State Transition Attributes */
 644         struct {
 645                 /* Autonmous Power State Transitions supported */
 646                 uint8_t         apst_supp : 1;
 647
 648                 uint8_t         apsta_rsvd : 7;
 649         } __packed apsta;
 650
 651         /** Warning Composite Temperature Threshold */
 652         uint16_t                wctemp;
 653
 654         /** Critical Composite Temperature Threshold */
 655         uint16_t                cctemp;
 656
 657         /** Maximum Time for Firmware Activation */
 658         uint16_t                mtfa;
 659
 660         /** Host Memory Buffer Preferred Size */
 661         uint32_t                hmpre;
 662
 663         /** Host Memory Buffer Minimum Size */
 664         uint32_t                hmmin;
 665
 666         /** Name space capabilities  */
 667         struct {
 668                 /* if nsmgmt, report tnvmcap and unvmcap */
 669                 uint8_t    tnvmcap[16];
 670                 uint8_t    unvmcap[16];
 671         } __packed untncap;
 672
 673         /** Replay Protected Memory Block Support */
 674         uint32_t                rpmbs; /* Really a bitfield */
 675
 676         /** Extended Device Self-test Time */
 677         uint16_t                edstt;
 678
 679         /** Device Self-test Options */
 680         uint8_t                 dsto; /* Really a bitfield */
 681
 682         /** Firmware Update Granularity */
 683         uint8_t                 fwug;
 684
 685         /** Keep Alive Support */
 686         uint16_t                kas;
 687
 688         /** Host Controlled Thermal Management Attributes */
 689         uint16_t                hctma; /* Really a bitfield */
 690
 691         /** Minimum Thermal Management Temperature */
 692         uint16_t                mntmt;
 693
 694         /** Maximum Thermal Management Temperature */
 695         uint16_t                mxtmt;
 696
 697         /** Sanitize Capabilities */
 698         uint32_t                sanicap; /* Really a bitfield */
 699
 700         uint8_t reserved3[180];
 701         /* bytes 512-703: nvm command set attributes */
 702
 703         /** submission queue entry size */
 704         struct {
 705                 uint8_t         min : 4;
 706                 uint8_t         max : 4;
 707         } __packed sqes;
 708
 709         /** completion queue entry size */
 710         struct {
 711                 uint8_t         min : 4;
 712                 uint8_t         max : 4;
 713         } __packed cqes;
 714
 715         /** Maximum Outstanding Commands */
 716         uint16_t                maxcmd;
 717
 718         /** number of namespaces */
 719         uint32_t                nn;
 720
 721         /** optional nvm command support */
 722         struct {
 723                 uint16_t        compare : 1;
 724                 uint16_t        write_unc : 1;
 725                 uint16_t        dsm: 1;
 726                 uint16_t        reserved: 13;
 727         } __packed oncs;
 728
 729         /** fused operation support */
 730         uint16_t                fuses;
 731
 732         /** format nvm attributes */
 733         uint8_t                 fna;
 734
 735         /** volatile write cache */
 736         struct {
 737                 uint8_t         present : 1;
 738                 uint8_t         reserved : 7;
 739         } __packed vwc;
 740
 741         /* TODO: flesh out remaining nvm command set attributes */
 742         uint8_t                 reserved5[178];
 743
 744         /* bytes 704-2047: i/o command set attributes */
 745         uint8_t                 reserved6[1344];
 746
 747         /* bytes 2048-3071: power state descriptors */
 748         struct nvme_power_state power_state[32];
 749
 750         /* bytes 3072-4095: vendor specific */
 751         uint8_t                 vs[1024];
 752 } __packed __aligned(4);
 753
 754 _Static_assert(sizeof(struct nvme_controller_data) == 4096, "bad size for nvme_controller_data");
 755
 756 struct nvme_namespace_data {
 757
 758         /** namespace size */
 759         uint64_t                nsze;
 760
 761         /** namespace capacity */
 762         uint64_t                ncap;
 763
 764         /** namespace utilization */
 765         uint64_t                nuse;
 766
 767         /** namespace features */
 768         struct {
 769                 /** thin provisioning */
 770                 uint8_t         thin_prov : 1;
 771                 uint8_t         reserved1 : 7;
 772         } __packed nsfeat;
 773
 774         /** number of lba formats */
 775         uint8_t                 nlbaf;
 776
 777         /** formatted lba size */
 778         struct {
 779                 uint8_t         format    : 4;
 780                 uint8_t         extended  : 1;
 781                 uint8_t         reserved2 : 3;
 782         } __packed flbas;
 783
 784         /** metadata capabilities */
 785         struct {
 786                 /* metadata can be transferred as part of data prp list */
 787                 uint8_t         extended  : 1;
 788
 789                 /* metadata can be transferred with separate metadata pointer */
 790                 uint8_t         pointer   : 1;
 791
 792                 uint8_t         reserved3 : 6;
 793         } __packed mc;
 794
 795         /** end-to-end data protection capabilities */
 796         struct {
 797                 /* protection information type 1 */
 798                 uint8_t         pit1     : 1;
 799
 800                 /* protection information type 2 */
 801                 uint8_t         pit2     : 1;
 802
 803                 /* protection information type 3 */
 804                 uint8_t         pit3     : 1;
 805
 806                 /* first eight bytes of metadata */
 807                 uint8_t         md_start : 1;
 808
 809                 /* last eight bytes of metadata */
 810                 uint8_t         md_end   : 1;
 811         } __packed dpc;
 812
 813         /** end-to-end data protection type settings */
 814         struct {
 815                 /* protection information type */
 816                 uint8_t         pit       : 3;
 817
 818                 /* 1 == protection info transferred at start of metadata */
 819                 /* 0 == protection info transferred at end of metadata */
 820                 uint8_t         md_start  : 1;
 821
 822                 uint8_t         reserved4 : 4;
 823         } __packed dps;
 824
 825         uint8_t                 reserved5[98];
 826
 827         /** lba format support */
 828         struct {
 829                 /** metadata size */
 830                 uint32_t        ms        : 16;
 831
 832                 /** lba data size */
 833                 uint32_t        lbads     : 8;
 834
 835                 /** relative performance */
 836                 uint32_t        rp        : 2;
 837
 838                 uint32_t        reserved6 : 6;
 839         } __packed lbaf[16];
 840
 841         uint8_t                 reserved6[192];
 842
 843         uint8_t                 vendor_specific[3712];
 844 } __packed __aligned(4);
 845
 846 _Static_assert(sizeof(struct nvme_namespace_data) == 4096, "bad size for nvme_namepsace_data");
 847
 848 enum nvme_log_page {
 849
 850         /* 0x00 - reserved */
 851         NVME_LOG_ERROR                  = 0x01,
 852         NVME_LOG_HEALTH_INFORMATION     = 0x02,
 853         NVME_LOG_FIRMWARE_SLOT          = 0x03,
 854         NVME_LOG_CHANGED_NAMESPACE      = 0x04,
 855         NVME_LOG_COMMAND_EFFECT         = 0x05,
 856         /* 0x06-0x7F - reserved */
 857         /* 0x80-0xBF - I/O command set specific */
 858         NVME_LOG_RES_NOTIFICATION       = 0x80,
 859         /* 0xC0-0xFF - vendor specific */
 860
 861         /*
 862          * The following are Intel Specific log pages, but they seem
 863          * to be widely implemented.
 864          */
 865         INTEL_LOG_READ_LAT_LOG          = 0xc1,
 866         INTEL_LOG_WRITE_LAT_LOG         = 0xc2,
 867         INTEL_LOG_TEMP_STATS            = 0xc5,
 868         INTEL_LOG_ADD_SMART             = 0xca,
 869         INTEL_LOG_DRIVE_MKT_NAME        = 0xdd,
 870
 871         /*
 872          * HGST log page, with lots ofs sub pages.
 873          */
 874         HGST_INFO_LOG                   = 0xc1,
 875 };
 876
 877 struct nvme_error_information_entry {
 878
 879         uint64_t                error_count;
 880         uint16_t                sqid;
 881         uint16_t                cid;
 882         struct nvme_status      status;
 883         uint16_t                error_location;
 884         uint64_t                lba;
 885         uint32_t                nsid;
 886         uint8_t                 vendor_specific;
 887         uint8_t                 reserved[35];
 888 } __packed __aligned(4);
 889
 890 _Static_assert(sizeof(struct nvme_error_information_entry) == 64, "bad size for nvme_error_information_entry");
 891
 892 union nvme_critical_warning_state {
 893
 894         uint8_t         raw;
 895
 896         struct {
 897                 uint8_t available_spare         : 1;
 898                 uint8_t temperature             : 1;
 899                 uint8_t device_reliability      : 1;
 900                 uint8_t read_only               : 1;
 901                 uint8_t volatile_memory_backup  : 1;
 902                 uint8_t reserved                : 3;
 903         } __packed bits;
 904 } __packed;
 905
 906 _Static_assert(sizeof(union nvme_critical_warning_state) == 1, "bad size for nvme_critical_warning_state");
 907
 908 struct nvme_health_information_page {
 909
 910         union nvme_critical_warning_state       critical_warning;
 911
 912         uint16_t                temperature;
 913         uint8_t                 available_spare;
 914         uint8_t                 available_spare_threshold;
 915         uint8_t                 percentage_used;
 916
 917         uint8_t                 reserved[26];
 918
 919         /*
 920          * Note that the following are 128-bit values, but are
 921          *  defined as an array of 2 64-bit values.
 922          */
 923         /* Data Units Read is always in 512-byte units. */
 924         uint64_t                data_units_read[2];
 925         /* Data Units Written is always in 512-byte units. */
 926         uint64_t                data_units_written[2];
 927         /* For NVM command set, this includes Compare commands. */
 928         uint64_t                host_read_commands[2];
 929         uint64_t                host_write_commands[2];
 930         /* Controller Busy Time is reported in minutes. */
 931         uint64_t                controller_busy_time[2];
 932         uint64_t                power_cycles[2];
 933         uint64_t                power_on_hours[2];
 934         uint64_t                unsafe_shutdowns[2];
 935         uint64_t                media_errors[2];
 936         uint64_t                num_error_info_log_entries[2];
 937         uint32_t                warning_temp_time;
 938         uint32_t                error_temp_time;
 939         uint16_t                temp_sensor[8];
 940
 941         uint8_t                 reserved2[296];
 942 } __packed __aligned(4);
 943
 944 _Static_assert(sizeof(struct nvme_health_information_page) == 512, "bad size for nvme_health_information_page");
 945
 946 struct nvme_firmware_page {
 947
 948         struct {
 949                 uint8_t slot            : 3; /* slot for current FW */
 950                 uint8_t reserved        : 5;
 951         } __packed afi;
 952
 953         uint8_t                 reserved[7];
 954         uint64_t                revision[7]; /* revisions for 7 slots */
 955         uint8_t                 reserved2[448];
 956 } __packed __aligned(4);
 957
 958 _Static_assert(sizeof(struct nvme_firmware_page) == 512, "bad size for nvme_firmware_page");
 959
 960 struct intel_log_temp_stats
 961 {
 962         uint64_t        current;
 963         uint64_t        overtemp_flag_last;
 964         uint64_t        overtemp_flag_life;
 965         uint64_t        max_temp;
 966         uint64_t        min_temp;
 967         uint64_t        _rsvd[5];
 968         uint64_t        max_oper_temp;
 969         uint64_t        min_oper_temp;
 970         uint64_t        est_offset;
 971 } __packed __aligned(4);
 972
 973 _Static_assert(sizeof(struct intel_log_temp_stats) == 13 * 8, "bad size for intel_log_temp_stats");
 974
 975 #define NVME_TEST_MAX_THREADS   128
 976
 977 struct nvme_io_test {
 978
 979         enum nvme_nvm_opcode    opc;
 980         uint32_t                size;
 981         uint32_t                time;   /* in seconds */
 982         uint32_t                num_threads;
 983         uint32_t                flags;
 984         uint64_t                io_completed[NVME_TEST_MAX_THREADS];
 985 };
 986
 987 enum nvme_io_test_flags {
 988
 989         /*
 990          * Specifies whether dev_refthread/dev_relthread should be
 991          *  called during NVME_BIO_TEST.  Ignored for other test
 992          *  types.
 993          */
 994         NVME_TEST_FLAG_REFTHREAD =      0x1,
 995 };
 996
 997 struct nvme_pt_command {
 998
 999         /*
1000          * cmd is used to specify a passthrough command to a controller or
1001          *  namespace.
1002          *
1003          * The following fields from cmd may be specified by the caller:
1004          *      * opc  (opcode)
1005          *      * nsid (namespace id) - for admin commands only
1006          *      * cdw10-cdw15
1007          *
1008          * Remaining fields must be set to 0 by the caller.
1009          */
1010         struct nvme_command     cmd;
1011
1012         /*
1013          * cpl returns completion status for the passthrough command
1014          *  specified by cmd.
1015          *
1016          * The following fields will be filled out by the driver, for
1017          *  consumption by the caller:
1018          *      * cdw0
1019          *      * status (except for phase)
1020          *
1021          * Remaining fields will be set to 0 by the driver.
1022          */
1023         struct nvme_completion  cpl;
1024
1025         /* buf is the data buffer associated with this passthrough command. */
1026         void *                  buf;
1027
1028         /*
1029          * len is the length of the data buffer associated with this
1030          *  passthrough command.
1031          */
1032         uint32_t                len;
1033
1034         /*
1035          * is_read = 1 if the passthrough command will read data into the
1036          *  supplied buffer from the controller.
1037          *
1038          * is_read = 0 if the passthrough command will write data from the
1039          *  supplied buffer to the controller.
1040          */
1041         uint32_t                is_read;
1042
1043         /*
1044          * driver_lock is used by the driver only.  It must be set to 0
1045          *  by the caller.
1046          */
1047         struct mtx *            driver_lock;
1048 };
1049
1050 #define nvme_completion_is_error(cpl)                                   \
1051         ((cpl)->status.sc != 0 || (cpl)->status.sct != 0)
1052
1053 void    nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen);
1054
1055 #ifdef _KERNEL
1056
1057 struct bio;
1058
1059 struct nvme_namespace;
1060 struct nvme_controller;
1061 struct nvme_consumer;
1062
1063 typedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *);
1064
1065 typedef void *(*nvme_cons_ns_fn_t)(struct nvme_namespace *, void *);
1066 typedef void *(*nvme_cons_ctrlr_fn_t)(struct nvme_controller *);
1067 typedef void (*nvme_cons_async_fn_t)(void *, const struct nvme_completion *,
1068                                      uint32_t, void *, uint32_t);
1069 typedef void (*nvme_cons_fail_fn_t)(void *);
1070
1071 enum nvme_namespace_flags {
1072         NVME_NS_DEALLOCATE_SUPPORTED    = 0x1,
1073         NVME_NS_FLUSH_SUPPORTED         = 0x2,
1074 };
1075
1076 int     nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
1077                                    struct nvme_pt_command *pt,
1078                                    uint32_t nsid, int is_user_buffer,
1079                                    int is_admin_cmd);
1080
1081 /* Admin functions */
1082 void    nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
1083                                    uint8_t feature, uint32_t cdw11,
1084                                    void *payload, uint32_t payload_size,
1085                                    nvme_cb_fn_t cb_fn, void *cb_arg);
1086 void    nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr,
1087                                    uint8_t feature, uint32_t cdw11,
1088                                    void *payload, uint32_t payload_size,
1089                                    nvme_cb_fn_t cb_fn, void *cb_arg);
1090 void    nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr,
1091                                     uint8_t log_page, uint32_t nsid,
1092                                     void *payload, uint32_t payload_size,
1093                                     nvme_cb_fn_t cb_fn, void *cb_arg);
1094
1095 /* NVM I/O functions */
1096 int     nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload,
1097                           uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
1098                           void *cb_arg);
1099 int     nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp,
1100                               nvme_cb_fn_t cb_fn, void *cb_arg);
1101 int     nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload,
1102                          uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
1103                          void *cb_arg);
1104 int     nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp,
1105                               nvme_cb_fn_t cb_fn, void *cb_arg);
1106 int     nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
1107                                uint8_t num_ranges, nvme_cb_fn_t cb_fn,
1108                                void *cb_arg);
1109 int     nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn,
1110                           void *cb_arg);
1111 int     nvme_ns_dump(struct nvme_namespace *ns, void *virt, off_t offset,
1112                      size_t len);
1113
1114 /* Registration functions */
1115 struct nvme_consumer *  nvme_register_consumer(nvme_cons_ns_fn_t    ns_fn,
1116                                                nvme_cons_ctrlr_fn_t ctrlr_fn,
1117                                                nvme_cons_async_fn_t async_fn,
1118                                                nvme_cons_fail_fn_t  fail_fn);
1119 void            nvme_unregister_consumer(struct nvme_consumer *consumer);
1120
1121 /* Controller helper functions */
1122 device_t        nvme_ctrlr_get_device(struct nvme_controller *ctrlr);
1123 const struct nvme_controller_data *
1124                 nvme_ctrlr_get_data(struct nvme_controller *ctrlr);
1125
1126 /* Namespace helper functions */
1127 uint32_t        nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns);
1128 uint32_t        nvme_ns_get_sector_size(struct nvme_namespace *ns);
1129 uint64_t        nvme_ns_get_num_sectors(struct nvme_namespace *ns);
1130 uint64_t        nvme_ns_get_size(struct nvme_namespace *ns);
1131 uint32_t        nvme_ns_get_flags(struct nvme_namespace *ns);
1132 const char *    nvme_ns_get_serial_number(struct nvme_namespace *ns);
1133 const char *    nvme_ns_get_model_number(struct nvme_namespace *ns);
1134 const struct nvme_namespace_data *
1135                 nvme_ns_get_data(struct nvme_namespace *ns);
1136 uint32_t        nvme_ns_get_stripesize(struct nvme_namespace *ns);
1137
1138 int     nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp,
1139                             nvme_cb_fn_t cb_fn);
1140
1141 /*
1142  * Command building helper functions -- shared with CAM
1143  * These functions assume allocator zeros out cmd structure
1144  * CAM's xpt_get_ccb and the request allocator for nvme both
1145  * do zero'd allocations.
1146  */
1147 static inline
1148 void    nvme_ns_flush_cmd(struct nvme_command *cmd, uint32_t nsid)
1149 {
1150
1151         cmd->opc = NVME_OPC_FLUSH;
1152         cmd->nsid = nsid;
1153 }
1154
1155 static inline
1156 void    nvme_ns_rw_cmd(struct nvme_command *cmd, uint32_t rwcmd, uint32_t nsid,
1157     uint64_t lba, uint32_t count)
1158 {
1159         cmd->opc = rwcmd;
1160         cmd->nsid = nsid;
1161         cmd->cdw10 = lba & 0xffffffffu;
1162         cmd->cdw11 = lba >> 32;
1163         cmd->cdw12 = count-1;
1164 }
1165
1166 static inline
1167 void    nvme_ns_write_cmd(struct nvme_command *cmd, uint32_t nsid,
1168     uint64_t lba, uint32_t count)
1169 {
1170         nvme_ns_rw_cmd(cmd, NVME_OPC_WRITE, nsid, lba, count);
1171 }
1172
1173 static inline
1174 void    nvme_ns_read_cmd(struct nvme_command *cmd, uint32_t nsid,
1175     uint64_t lba, uint32_t count)
1176 {
1177         nvme_ns_rw_cmd(cmd, NVME_OPC_READ, nsid, lba, count);
1178 }
1179
1180 static inline
1181 void    nvme_ns_trim_cmd(struct nvme_command *cmd, uint32_t nsid,
1182     uint32_t num_ranges)
1183 {
1184         cmd->opc = NVME_OPC_DATASET_MANAGEMENT;
1185         cmd->nsid = nsid;
1186         cmd->cdw10 = num_ranges - 1;
1187         cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE;
1188 }
1189
1190 extern int nvme_use_nvd;
1191
1192 #endif /* _KERNEL */
1193
1194 #endif /* __NVME_H__ */