sys/dev/nvme/nvme.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (C) 2012-2013 Intel Corporation
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  *
  28  * $FreeBSD$
  29  */
  30
  31 #ifndef __NVME_H__
  32 #define __NVME_H__
  33
  34 #ifdef _KERNEL
  35 #include <sys/types.h>
  36 #endif
  37
  38 #include <sys/param.h>
  39 #include <sys/endian.h>
  40
  41 #define NVME_PASSTHROUGH_CMD            _IOWR('n', 0, struct nvme_pt_command)
  42 #define NVME_RESET_CONTROLLER           _IO('n', 1)
  43
  44 #define NVME_IO_TEST                    _IOWR('n', 100, struct nvme_io_test)
  45 #define NVME_BIO_TEST                   _IOWR('n', 101, struct nvme_io_test)
  46
  47 /*
  48  * Macros to deal with NVME revisions, as defined VS register
  49  */
  50 #define NVME_REV(x, y)                  (((x) << 16) | ((y) << 8))
  51 #define NVME_MAJOR(r)                   (((r) >> 16) & 0xffff)
  52 #define NVME_MINOR(r)                   (((r) >> 8) & 0xff)
  53
  54 /*
  55  * Use to mark a command to apply to all namespaces, or to retrieve global
  56  *  log pages.
  57  */
  58 #define NVME_GLOBAL_NAMESPACE_TAG       ((uint32_t)0xFFFFFFFF)
  59
  60 /* Cap nvme to 1MB transfers driver explodes with larger sizes */
  61 #define NVME_MAX_XFER_SIZE              (MAXPHYS < (1<<20) ? MAXPHYS : (1<<20))
  62
  63 /* Register field definitions */
  64 #define NVME_CAP_LO_REG_MQES_SHIFT                      (0)
  65 #define NVME_CAP_LO_REG_MQES_MASK                       (0xFFFF)
  66 #define NVME_CAP_LO_REG_CQR_SHIFT                       (16)
  67 #define NVME_CAP_LO_REG_CQR_MASK                        (0x1)
  68 #define NVME_CAP_LO_REG_AMS_SHIFT                       (17)
  69 #define NVME_CAP_LO_REG_AMS_MASK                        (0x3)
  70 #define NVME_CAP_LO_REG_TO_SHIFT                        (24)
  71 #define NVME_CAP_LO_REG_TO_MASK                         (0xFF)
  72
  73 #define NVME_CAP_HI_REG_DSTRD_SHIFT                     (0)
  74 #define NVME_CAP_HI_REG_DSTRD_MASK                      (0xF)
  75 #define NVME_CAP_HI_REG_CSS_NVM_SHIFT                   (5)
  76 #define NVME_CAP_HI_REG_CSS_NVM_MASK                    (0x1)
  77 #define NVME_CAP_HI_REG_MPSMIN_SHIFT                    (16)
  78 #define NVME_CAP_HI_REG_MPSMIN_MASK                     (0xF)
  79 #define NVME_CAP_HI_REG_MPSMAX_SHIFT                    (20)
  80 #define NVME_CAP_HI_REG_MPSMAX_MASK                     (0xF)
  81
  82 #define NVME_CC_REG_EN_SHIFT                            (0)
  83 #define NVME_CC_REG_EN_MASK                             (0x1)
  84 #define NVME_CC_REG_CSS_SHIFT                           (4)
  85 #define NVME_CC_REG_CSS_MASK                            (0x7)
  86 #define NVME_CC_REG_MPS_SHIFT                           (7)
  87 #define NVME_CC_REG_MPS_MASK                            (0xF)
  88 #define NVME_CC_REG_AMS_SHIFT                           (11)
  89 #define NVME_CC_REG_AMS_MASK                            (0x7)
  90 #define NVME_CC_REG_SHN_SHIFT                           (14)
  91 #define NVME_CC_REG_SHN_MASK                            (0x3)
  92 #define NVME_CC_REG_IOSQES_SHIFT                        (16)
  93 #define NVME_CC_REG_IOSQES_MASK                         (0xF)
  94 #define NVME_CC_REG_IOCQES_SHIFT                        (20)
  95 #define NVME_CC_REG_IOCQES_MASK                         (0xF)
  96
  97 #define NVME_CSTS_REG_RDY_SHIFT                         (0)
  98 #define NVME_CSTS_REG_RDY_MASK                          (0x1)
  99 #define NVME_CSTS_REG_CFS_SHIFT                         (1)
 100 #define NVME_CSTS_REG_CFS_MASK                          (0x1)
 101 #define NVME_CSTS_REG_SHST_SHIFT                        (2)
 102 #define NVME_CSTS_REG_SHST_MASK                         (0x3)
 103
 104 #define NVME_CSTS_GET_SHST(csts)                        (((csts) >> NVME_CSTS_REG_SHST_SHIFT) & NVME_CSTS_REG_SHST_MASK)
 105
 106 #define NVME_AQA_REG_ASQS_SHIFT                         (0)
 107 #define NVME_AQA_REG_ASQS_MASK                          (0xFFF)
 108 #define NVME_AQA_REG_ACQS_SHIFT                         (16)
 109 #define NVME_AQA_REG_ACQS_MASK                          (0xFFF)
 110
 111 /* Command field definitions */
 112
 113 #define NVME_CMD_FUSE_SHIFT                             (8)
 114 #define NVME_CMD_FUSE_MASK                              (0x3)
 115
 116 #define NVME_STATUS_P_SHIFT                             (0)
 117 #define NVME_STATUS_P_MASK                              (0x1)
 118 #define NVME_STATUS_SC_SHIFT                            (1)
 119 #define NVME_STATUS_SC_MASK                             (0xFF)
 120 #define NVME_STATUS_SCT_SHIFT                           (9)
 121 #define NVME_STATUS_SCT_MASK                            (0x7)
 122 #define NVME_STATUS_M_SHIFT                             (14)
 123 #define NVME_STATUS_M_MASK                              (0x1)
 124 #define NVME_STATUS_DNR_SHIFT                           (15)
 125 #define NVME_STATUS_DNR_MASK                            (0x1)
 126
 127 #define NVME_STATUS_GET_P(st)                           (((st) >> NVME_STATUS_P_SHIFT) & NVME_STATUS_P_MASK)
 128 #define NVME_STATUS_GET_SC(st)                          (((st) >> NVME_STATUS_SC_SHIFT) & NVME_STATUS_SC_MASK)
 129 #define NVME_STATUS_GET_SCT(st)                         (((st) >> NVME_STATUS_SCT_SHIFT) & NVME_STATUS_SCT_MASK)
 130 #define NVME_STATUS_GET_M(st)                           (((st) >> NVME_STATUS_M_SHIFT) & NVME_STATUS_M_MASK)
 131 #define NVME_STATUS_GET_DNR(st)                         (((st) >> NVME_STATUS_DNR_SHIFT) & NVME_STATUS_DNR_MASK)
 132
 133 #define NVME_PWR_ST_MPS_SHIFT                           (0)
 134 #define NVME_PWR_ST_MPS_MASK                            (0x1)
 135 #define NVME_PWR_ST_NOPS_SHIFT                          (1)
 136 #define NVME_PWR_ST_NOPS_MASK                           (0x1)
 137 #define NVME_PWR_ST_RRT_SHIFT                           (0)
 138 #define NVME_PWR_ST_RRT_MASK                            (0x1F)
 139 #define NVME_PWR_ST_RRL_SHIFT                           (0)
 140 #define NVME_PWR_ST_RRL_MASK                            (0x1F)
 141 #define NVME_PWR_ST_RWT_SHIFT                           (0)
 142 #define NVME_PWR_ST_RWT_MASK                            (0x1F)
 143 #define NVME_PWR_ST_RWL_SHIFT                           (0)
 144 #define NVME_PWR_ST_RWL_MASK                            (0x1F)
 145 #define NVME_PWR_ST_IPS_SHIFT                           (6)
 146 #define NVME_PWR_ST_IPS_MASK                            (0x3)
 147 #define NVME_PWR_ST_APW_SHIFT                           (0)
 148 #define NVME_PWR_ST_APW_MASK                            (0x7)
 149 #define NVME_PWR_ST_APS_SHIFT                           (6)
 150 #define NVME_PWR_ST_APS_MASK                            (0x3)
 151
 152 /** Controller Multi-path I/O and Namespace Sharing Capabilities */
 153 /* More then one port */
 154 #define NVME_CTRLR_DATA_MIC_MPORTS_SHIFT                (0)
 155 #define NVME_CTRLR_DATA_MIC_MPORTS_MASK                 (0x1)
 156 /* More then one controller */
 157 #define NVME_CTRLR_DATA_MIC_MCTRLRS_SHIFT               (1)
 158 #define NVME_CTRLR_DATA_MIC_MCTRLRS_MASK                (0x1)
 159 /* SR-IOV Virtual Function */
 160 #define NVME_CTRLR_DATA_MIC_SRIOVVF_SHIFT               (2)
 161 #define NVME_CTRLR_DATA_MIC_SRIOVVF_MASK                (0x1)
 162
 163 /** OACS - optional admin command support */
 164 /* supports security send/receive commands */
 165 #define NVME_CTRLR_DATA_OACS_SECURITY_SHIFT             (0)
 166 #define NVME_CTRLR_DATA_OACS_SECURITY_MASK              (0x1)
 167 /* supports format nvm command */
 168 #define NVME_CTRLR_DATA_OACS_FORMAT_SHIFT               (1)
 169 #define NVME_CTRLR_DATA_OACS_FORMAT_MASK                (0x1)
 170 /* supports firmware activate/download commands */
 171 #define NVME_CTRLR_DATA_OACS_FIRMWARE_SHIFT             (2)
 172 #define NVME_CTRLR_DATA_OACS_FIRMWARE_MASK              (0x1)
 173 /* supports namespace management commands */
 174 #define NVME_CTRLR_DATA_OACS_NSMGMT_SHIFT               (3)
 175 #define NVME_CTRLR_DATA_OACS_NSMGMT_MASK                (0x1)
 176 /* supports Device Self-test command */
 177 #define NVME_CTRLR_DATA_OACS_SELFTEST_SHIFT             (4)
 178 #define NVME_CTRLR_DATA_OACS_SELFTEST_MASK              (0x1)
 179 /* supports Directives */
 180 #define NVME_CTRLR_DATA_OACS_DIRECTIVES_SHIFT           (5)
 181 #define NVME_CTRLR_DATA_OACS_DIRECTIVES_MASK            (0x1)
 182 /* supports NVMe-MI Send/Receive */
 183 #define NVME_CTRLR_DATA_OACS_NVMEMI_SHIFT               (6)
 184 #define NVME_CTRLR_DATA_OACS_NVMEMI_MASK                (0x1)
 185 /* supports Virtualization Management */
 186 #define NVME_CTRLR_DATA_OACS_VM_SHIFT                   (7)
 187 #define NVME_CTRLR_DATA_OACS_VM_MASK                    (0x1)
 188 /* supports Doorbell Buffer Config */
 189 #define NVME_CTRLR_DATA_OACS_DBBUFFER_SHIFT             (8)
 190 #define NVME_CTRLR_DATA_OACS_DBBUFFER_MASK              (0x1)
 191
 192 /** firmware updates */
 193 /* first slot is read-only */
 194 #define NVME_CTRLR_DATA_FRMW_SLOT1_RO_SHIFT             (0)
 195 #define NVME_CTRLR_DATA_FRMW_SLOT1_RO_MASK              (0x1)
 196 /* number of firmware slots */
 197 #define NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT            (1)
 198 #define NVME_CTRLR_DATA_FRMW_NUM_SLOTS_MASK             (0x7)
 199
 200 /** log page attributes */
 201 /* per namespace smart/health log page */
 202 #define NVME_CTRLR_DATA_LPA_NS_SMART_SHIFT              (0)
 203 #define NVME_CTRLR_DATA_LPA_NS_SMART_MASK               (0x1)
 204
 205 /** AVSCC - admin vendor specific command configuration */
 206 /* admin vendor specific commands use spec format */
 207 #define NVME_CTRLR_DATA_AVSCC_SPEC_FORMAT_SHIFT         (0)
 208 #define NVME_CTRLR_DATA_AVSCC_SPEC_FORMAT_MASK          (0x1)
 209
 210 /** Autonomous Power State Transition Attributes */
 211 /* Autonomous Power State Transitions supported */
 212 #define NVME_CTRLR_DATA_APSTA_APST_SUPP_SHIFT           (0)
 213 #define NVME_CTRLR_DATA_APSTA_APST_SUPP_MASK            (0x1)
 214
 215 /** submission queue entry size */
 216 #define NVME_CTRLR_DATA_SQES_MIN_SHIFT                  (0)
 217 #define NVME_CTRLR_DATA_SQES_MIN_MASK                   (0xF)
 218 #define NVME_CTRLR_DATA_SQES_MAX_SHIFT                  (4)
 219 #define NVME_CTRLR_DATA_SQES_MAX_MASK                   (0xF)
 220
 221 /** completion queue entry size */
 222 #define NVME_CTRLR_DATA_CQES_MIN_SHIFT                  (0)
 223 #define NVME_CTRLR_DATA_CQES_MIN_MASK                   (0xF)
 224 #define NVME_CTRLR_DATA_CQES_MAX_SHIFT                  (4)
 225 #define NVME_CTRLR_DATA_CQES_MAX_MASK                   (0xF)
 226
 227 /** optional nvm command support */
 228 #define NVME_CTRLR_DATA_ONCS_COMPARE_SHIFT              (0)
 229 #define NVME_CTRLR_DATA_ONCS_COMPARE_MASK               (0x1)
 230 #define NVME_CTRLR_DATA_ONCS_WRITE_UNC_SHIFT            (1)
 231 #define NVME_CTRLR_DATA_ONCS_WRITE_UNC_MASK             (0x1)
 232 #define NVME_CTRLR_DATA_ONCS_DSM_SHIFT                  (2)
 233 #define NVME_CTRLR_DATA_ONCS_DSM_MASK                   (0x1)
 234 #define NVME_CTRLR_DATA_ONCS_WRZERO_SHIFT               (3)
 235 #define NVME_CTRLR_DATA_ONCS_WRZERO_MASK                (0x1)
 236 #define NVME_CTRLR_DATA_ONCS_SAVEFEAT_SHIFT             (4)
 237 #define NVME_CTRLR_DATA_ONCS_SAVEFEAT_MASK              (0x1)
 238 #define NVME_CTRLR_DATA_ONCS_RESERV_SHIFT               (5)
 239 #define NVME_CTRLR_DATA_ONCS_RESERV_MASK                (0x1)
 240 #define NVME_CTRLR_DATA_ONCS_TIMESTAMP_SHIFT            (6)
 241 #define NVME_CTRLR_DATA_ONCS_TIMESTAMP_MASK             (0x1)
 242
 243 /** Fused Operation Support */
 244 #define NVME_CTRLR_DATA_FUSES_CNW_SHIFT         (0)
 245 #define NVME_CTRLR_DATA_FUSES_CNW_MASK          (0x1)
 246
 247 /** Format NVM Attributes */
 248 #define NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT            (0)
 249 #define NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK             (0x1)
 250 #define NVME_CTRLR_DATA_FNA_ERASE_ALL_SHIFT             (1)
 251 #define NVME_CTRLR_DATA_FNA_ERASE_ALL_MASK              (0x1)
 252 #define NVME_CTRLR_DATA_FNA_CRYPTO_ERASE_SHIFT          (2)
 253 #define NVME_CTRLR_DATA_FNA_CRYPTO_ERASE_MASK           (0x1)
 254
 255 /** volatile write cache */
 256 #define NVME_CTRLR_DATA_VWC_PRESENT_SHIFT               (0)
 257 #define NVME_CTRLR_DATA_VWC_PRESENT_MASK                (0x1)
 258
 259 /** namespace features */
 260 /* thin provisioning */
 261 #define NVME_NS_DATA_NSFEAT_THIN_PROV_SHIFT             (0)
 262 #define NVME_NS_DATA_NSFEAT_THIN_PROV_MASK              (0x1)
 263 /* NAWUN, NAWUPF, and NACWU fields are valid */
 264 #define NVME_NS_DATA_NSFEAT_NA_FIELDS_SHIFT             (1)
 265 #define NVME_NS_DATA_NSFEAT_NA_FIELDS_MASK              (0x1)
 266 /* Deallocated or Unwritten Logical Block errors supported */
 267 #define NVME_NS_DATA_NSFEAT_DEALLOC_SHIFT               (2)
 268 #define NVME_NS_DATA_NSFEAT_DEALLOC_MASK                (0x1)
 269 /* NGUID and EUI64 fields are not reusable */
 270 #define NVME_NS_DATA_NSFEAT_NO_ID_REUSE_SHIFT           (3)
 271 #define NVME_NS_DATA_NSFEAT_NO_ID_REUSE_MASK            (0x1)
 272
 273 /** formatted lba size */
 274 #define NVME_NS_DATA_FLBAS_FORMAT_SHIFT                 (0)
 275 #define NVME_NS_DATA_FLBAS_FORMAT_MASK                  (0xF)
 276 #define NVME_NS_DATA_FLBAS_EXTENDED_SHIFT               (4)
 277 #define NVME_NS_DATA_FLBAS_EXTENDED_MASK                (0x1)
 278
 279 /** metadata capabilities */
 280 /* metadata can be transferred as part of data prp list */
 281 #define NVME_NS_DATA_MC_EXTENDED_SHIFT                  (0)
 282 #define NVME_NS_DATA_MC_EXTENDED_MASK                   (0x1)
 283 /* metadata can be transferred with separate metadata pointer */
 284 #define NVME_NS_DATA_MC_POINTER_SHIFT                   (1)
 285 #define NVME_NS_DATA_MC_POINTER_MASK                    (0x1)
 286
 287 /** end-to-end data protection capabilities */
 288 /* protection information type 1 */
 289 #define NVME_NS_DATA_DPC_PIT1_SHIFT                     (0)
 290 #define NVME_NS_DATA_DPC_PIT1_MASK                      (0x1)
 291 /* protection information type 2 */
 292 #define NVME_NS_DATA_DPC_PIT2_SHIFT                     (1)
 293 #define NVME_NS_DATA_DPC_PIT2_MASK                      (0x1)
 294 /* protection information type 3 */
 295 #define NVME_NS_DATA_DPC_PIT3_SHIFT                     (2)
 296 #define NVME_NS_DATA_DPC_PIT3_MASK                      (0x1)
 297 /* first eight bytes of metadata */
 298 #define NVME_NS_DATA_DPC_MD_START_SHIFT                 (3)
 299 #define NVME_NS_DATA_DPC_MD_START_MASK                  (0x1)
 300 /* last eight bytes of metadata */
 301 #define NVME_NS_DATA_DPC_MD_END_SHIFT                   (4)
 302 #define NVME_NS_DATA_DPC_MD_END_MASK                    (0x1)
 303
 304 /** end-to-end data protection type settings */
 305 /* protection information type */
 306 #define NVME_NS_DATA_DPS_PIT_SHIFT                      (0)
 307 #define NVME_NS_DATA_DPS_PIT_MASK                       (0x7)
 308 /* 1 == protection info transferred at start of metadata */
 309 /* 0 == protection info transferred at end of metadata */
 310 #define NVME_NS_DATA_DPS_MD_START_SHIFT                 (3)
 311 #define NVME_NS_DATA_DPS_MD_START_MASK                  (0x1)
 312
 313 /** Namespace Multi-path I/O and Namespace Sharing Capabilities */
 314 /* the namespace may be attached to two or more controllers */
 315 #define NVME_NS_DATA_NMIC_MAY_BE_SHARED_SHIFT           (0)
 316 #define NVME_NS_DATA_NMIC_MAY_BE_SHARED_MASK            (0x1)
 317
 318 /** Reservation Capabilities */
 319 /* Persist Through Power Loss */
 320 #define NVME_NS_DATA_RESCAP_PTPL_SHIFT          (0)
 321 #define NVME_NS_DATA_RESCAP_PTPL_MASK           (0x1)
 322 /* supports the Write Exclusive */
 323 #define NVME_NS_DATA_RESCAP_WR_EX_SHIFT         (1)
 324 #define NVME_NS_DATA_RESCAP_WR_EX_MASK          (0x1)
 325 /* supports the Exclusive Access */
 326 #define NVME_NS_DATA_RESCAP_EX_AC_SHIFT         (2)
 327 #define NVME_NS_DATA_RESCAP_EX_AC_MASK          (0x1)
 328 /* supports the Write Exclusive – Registrants Only */
 329 #define NVME_NS_DATA_RESCAP_WR_EX_RO_SHIFT      (3)
 330 #define NVME_NS_DATA_RESCAP_WR_EX_RO_MASK       (0x1)
 331 /* supports the Exclusive Access - Registrants Only */
 332 #define NVME_NS_DATA_RESCAP_EX_AC_RO_SHIFT      (4)
 333 #define NVME_NS_DATA_RESCAP_EX_AC_RO_MASK       (0x1)
 334 /* supports the Write Exclusive – All Registrants */
 335 #define NVME_NS_DATA_RESCAP_WR_EX_AR_SHIFT      (5)
 336 #define NVME_NS_DATA_RESCAP_WR_EX_AR_MASK       (0x1)
 337 /* supports the Exclusive Access - All Registrants */
 338 #define NVME_NS_DATA_RESCAP_EX_AC_AR_SHIFT      (6)
 339 #define NVME_NS_DATA_RESCAP_EX_AC_AR_MASK       (0x1)
 340 /* Ignore Existing Key is used as defined in revision 1.3 or later */
 341 #define NVME_NS_DATA_RESCAP_IEKEY13_SHIFT       (7)
 342 #define NVME_NS_DATA_RESCAP_IEKEY13_MASK        (0x1)
 343
 344 /** Format Progress Indicator */
 345 /* percentage of the Format NVM command that remains to be completed */
 346 #define NVME_NS_DATA_FPI_PERC_SHIFT             (0)
 347 #define NVME_NS_DATA_FPI_PERC_MASK              (0x7f)
 348 /* namespace supports the Format Progress Indicator */
 349 #define NVME_NS_DATA_FPI_SUPP_SHIFT             (7)
 350 #define NVME_NS_DATA_FPI_SUPP_MASK              (0x1)
 351
 352 /** lba format support */
 353 /* metadata size */
 354 #define NVME_NS_DATA_LBAF_MS_SHIFT                      (0)
 355 #define NVME_NS_DATA_LBAF_MS_MASK                       (0xFFFF)
 356 /* lba data size */
 357 #define NVME_NS_DATA_LBAF_LBADS_SHIFT                   (16)
 358 #define NVME_NS_DATA_LBAF_LBADS_MASK                    (0xFF)
 359 /* relative performance */
 360 #define NVME_NS_DATA_LBAF_RP_SHIFT                      (24)
 361 #define NVME_NS_DATA_LBAF_RP_MASK                       (0x3)
 362
 363 enum nvme_critical_warning_state {
 364         NVME_CRIT_WARN_ST_AVAILABLE_SPARE               = 0x1,
 365         NVME_CRIT_WARN_ST_TEMPERATURE                   = 0x2,
 366         NVME_CRIT_WARN_ST_DEVICE_RELIABILITY            = 0x4,
 367         NVME_CRIT_WARN_ST_READ_ONLY                     = 0x8,
 368         NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP        = 0x10,
 369 };
 370 #define NVME_CRIT_WARN_ST_RESERVED_MASK                 (0xE0)
 371
 372 /* slot for current FW */
 373 #define NVME_FIRMWARE_PAGE_AFI_SLOT_SHIFT               (0)
 374 #define NVME_FIRMWARE_PAGE_AFI_SLOT_MASK                (0x7)
 375
 376 /* CC register SHN field values */
 377 enum shn_value {
 378         NVME_SHN_NORMAL         = 0x1,
 379         NVME_SHN_ABRUPT         = 0x2,
 380 };
 381
 382 /* CSTS register SHST field values */
 383 enum shst_value {
 384         NVME_SHST_NORMAL        = 0x0,
 385         NVME_SHST_OCCURRING     = 0x1,
 386         NVME_SHST_COMPLETE      = 0x2,
 387 };
 388
 389 struct nvme_registers
 390 {
 391         /** controller capabilities */
 392         uint32_t                cap_lo;
 393         uint32_t                cap_hi;
 394
 395         uint32_t                vs;     /* version */
 396         uint32_t                intms;  /* interrupt mask set */
 397         uint32_t                intmc;  /* interrupt mask clear */
 398
 399         /** controller configuration */
 400         uint32_t                cc;
 401
 402         uint32_t                reserved1;
 403
 404         /** controller status */
 405         uint32_t                csts;
 406
 407         uint32_t                reserved2;
 408
 409         /** admin queue attributes */
 410         uint32_t                aqa;
 411
 412         uint64_t                asq;    /* admin submission queue base addr */
 413         uint64_t                acq;    /* admin completion queue base addr */
 414         uint32_t                reserved3[0x3f2];
 415
 416         struct {
 417             uint32_t            sq_tdbl; /* submission queue tail doorbell */
 418             uint32_t            cq_hdbl; /* completion queue head doorbell */
 419         } doorbell[1] __packed;
 420 } __packed;
 421
 422 _Static_assert(sizeof(struct nvme_registers) == 0x1008, "bad size for nvme_registers");
 423
 424 struct nvme_command
 425 {
 426         /* dword 0 */
 427         uint8_t opc;            /* opcode */
 428         uint8_t fuse;           /* fused operation */
 429         uint16_t cid;           /* command identifier */
 430
 431         /* dword 1 */
 432         uint32_t nsid;          /* namespace identifier */
 433
 434         /* dword 2-3 */
 435         uint32_t rsvd2;
 436         uint32_t rsvd3;
 437
 438         /* dword 4-5 */
 439         uint64_t mptr;          /* metadata pointer */
 440
 441         /* dword 6-7 */
 442         uint64_t prp1;          /* prp entry 1 */
 443
 444         /* dword 8-9 */
 445         uint64_t prp2;          /* prp entry 2 */
 446
 447         /* dword 10-15 */
 448         uint32_t cdw10;         /* command-specific */
 449         uint32_t cdw11;         /* command-specific */
 450         uint32_t cdw12;         /* command-specific */
 451         uint32_t cdw13;         /* command-specific */
 452         uint32_t cdw14;         /* command-specific */
 453         uint32_t cdw15;         /* command-specific */
 454 } __packed;
 455
 456 _Static_assert(sizeof(struct nvme_command) == 16 * 4, "bad size for nvme_command");
 457
 458 struct nvme_completion {
 459
 460         /* dword 0 */
 461         uint32_t                cdw0;   /* command-specific */
 462
 463         /* dword 1 */
 464         uint32_t                rsvd1;
 465
 466         /* dword 2 */
 467         uint16_t                sqhd;   /* submission queue head pointer */
 468         uint16_t                sqid;   /* submission queue identifier */
 469
 470         /* dword 3 */
 471         uint16_t                cid;    /* command identifier */
 472         uint16_t                status;
 473 } __packed;
 474
 475 _Static_assert(sizeof(struct nvme_completion) == 4 * 4, "bad size for nvme_completion");
 476
 477 struct nvme_dsm_range {
 478         uint32_t attributes;
 479         uint32_t length;
 480         uint64_t starting_lba;
 481 } __packed;
 482
 483 /* Largest DSM Trim that can be done */
 484 #define NVME_MAX_DSM_TRIM               4096
 485
 486 _Static_assert(sizeof(struct nvme_dsm_range) == 16, "bad size for nvme_dsm_ranage");
 487
 488 /* status code types */
 489 enum nvme_status_code_type {
 490         NVME_SCT_GENERIC                = 0x0,
 491         NVME_SCT_COMMAND_SPECIFIC       = 0x1,
 492         NVME_SCT_MEDIA_ERROR            = 0x2,
 493         /* 0x3-0x6 - reserved */
 494         NVME_SCT_VENDOR_SPECIFIC        = 0x7,
 495 };
 496
 497 /* generic command status codes */
 498 enum nvme_generic_command_status_code {
 499         NVME_SC_SUCCESS                         = 0x00,
 500         NVME_SC_INVALID_OPCODE                  = 0x01,
 501         NVME_SC_INVALID_FIELD                   = 0x02,
 502         NVME_SC_COMMAND_ID_CONFLICT             = 0x03,
 503         NVME_SC_DATA_TRANSFER_ERROR             = 0x04,
 504         NVME_SC_ABORTED_POWER_LOSS              = 0x05,
 505         NVME_SC_INTERNAL_DEVICE_ERROR           = 0x06,
 506         NVME_SC_ABORTED_BY_REQUEST              = 0x07,
 507         NVME_SC_ABORTED_SQ_DELETION             = 0x08,
 508         NVME_SC_ABORTED_FAILED_FUSED            = 0x09,
 509         NVME_SC_ABORTED_MISSING_FUSED           = 0x0a,
 510         NVME_SC_INVALID_NAMESPACE_OR_FORMAT     = 0x0b,
 511         NVME_SC_COMMAND_SEQUENCE_ERROR          = 0x0c,
 512         NVME_SC_INVALID_SGL_SEGMENT_DESCR       = 0x0d,
 513         NVME_SC_INVALID_NUMBER_OF_SGL_DESCR     = 0x0e,
 514         NVME_SC_DATA_SGL_LENGTH_INVALID         = 0x0f,
 515         NVME_SC_METADATA_SGL_LENGTH_INVALID     = 0x10,
 516         NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID     = 0x11,
 517         NVME_SC_INVALID_USE_OF_CMB              = 0x12,
 518         NVME_SC_PRP_OFFET_INVALID               = 0x13,
 519         NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED      = 0x14,
 520         NVME_SC_OPERATION_DENIED                = 0x15,
 521         NVME_SC_SGL_OFFSET_INVALID              = 0x16,
 522         /* 0x17 - reserved */
 523         NVME_SC_HOST_ID_INCONSISTENT_FORMAT     = 0x18,
 524         NVME_SC_KEEP_ALIVE_TIMEOUT_EXPIRED      = 0x19,
 525         NVME_SC_KEEP_ALIVE_TIMEOUT_INVALID      = 0x1a,
 526         NVME_SC_ABORTED_DUE_TO_PREEMPT          = 0x1b,
 527         NVME_SC_SANITIZE_FAILED                 = 0x1c,
 528         NVME_SC_SANITIZE_IN_PROGRESS            = 0x1d,
 529         NVME_SC_SGL_DATA_BLOCK_GRAN_INVALID     = 0x1e,
 530         NVME_SC_NOT_SUPPORTED_IN_CMB            = 0x1f,
 531
 532         NVME_SC_LBA_OUT_OF_RANGE                = 0x80,
 533         NVME_SC_CAPACITY_EXCEEDED               = 0x81,
 534         NVME_SC_NAMESPACE_NOT_READY             = 0x82,
 535         NVME_SC_RESERVATION_CONFLICT            = 0x83,
 536         NVME_SC_FORMAT_IN_PROGRESS              = 0x84,
 537 };
 538
 539 /* command specific status codes */
 540 enum nvme_command_specific_status_code {
 541         NVME_SC_COMPLETION_QUEUE_INVALID        = 0x00,
 542         NVME_SC_INVALID_QUEUE_IDENTIFIER        = 0x01,
 543         NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED     = 0x02,
 544         NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED    = 0x03,
 545         /* 0x04 - reserved */
 546         NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED = 0x05,
 547         NVME_SC_INVALID_FIRMWARE_SLOT           = 0x06,
 548         NVME_SC_INVALID_FIRMWARE_IMAGE          = 0x07,
 549         NVME_SC_INVALID_INTERRUPT_VECTOR        = 0x08,
 550         NVME_SC_INVALID_LOG_PAGE                = 0x09,
 551         NVME_SC_INVALID_FORMAT                  = 0x0a,
 552         NVME_SC_FIRMWARE_REQUIRES_RESET         = 0x0b,
 553         NVME_SC_INVALID_QUEUE_DELETION          = 0x0c,
 554         NVME_SC_FEATURE_NOT_SAVEABLE            = 0x0d,
 555         NVME_SC_FEATURE_NOT_CHANGEABLE          = 0x0e,
 556         NVME_SC_FEATURE_NOT_NS_SPECIFIC         = 0x0f,
 557         NVME_SC_FW_ACT_REQUIRES_NVMS_RESET      = 0x10,
 558         NVME_SC_FW_ACT_REQUIRES_RESET           = 0x11,
 559         NVME_SC_FW_ACT_REQUIRES_TIME            = 0x12,
 560         NVME_SC_FW_ACT_PROHIBITED               = 0x13,
 561         NVME_SC_OVERLAPPING_RANGE               = 0x14,
 562         NVME_SC_NS_INSUFFICIENT_CAPACITY        = 0x15,
 563         NVME_SC_NS_ID_UNAVAILABLE               = 0x16,
 564         /* 0x17 - reserved */
 565         NVME_SC_NS_ALREADY_ATTACHED             = 0x18,
 566         NVME_SC_NS_IS_PRIVATE                   = 0x19,
 567         NVME_SC_NS_NOT_ATTACHED                 = 0x1a,
 568         NVME_SC_THIN_PROV_NOT_SUPPORTED         = 0x1b,
 569         NVME_SC_CTRLR_LIST_INVALID              = 0x1c,
 570         NVME_SC_SELT_TEST_IN_PROGRESS           = 0x1d,
 571         NVME_SC_BOOT_PART_WRITE_PROHIB          = 0x1e,
 572         NVME_SC_INVALID_CTRLR_ID                = 0x1f,
 573         NVME_SC_INVALID_SEC_CTRLR_STATE         = 0x20,
 574         NVME_SC_INVALID_NUM_OF_CTRLR_RESRC      = 0x21,
 575         NVME_SC_INVALID_RESOURCE_ID             = 0x22,
 576
 577         NVME_SC_CONFLICTING_ATTRIBUTES          = 0x80,
 578         NVME_SC_INVALID_PROTECTION_INFO         = 0x81,
 579         NVME_SC_ATTEMPTED_WRITE_TO_RO_PAGE      = 0x82,
 580 };
 581
 582 /* media error status codes */
 583 enum nvme_media_error_status_code {
 584         NVME_SC_WRITE_FAULTS                    = 0x80,
 585         NVME_SC_UNRECOVERED_READ_ERROR          = 0x81,
 586         NVME_SC_GUARD_CHECK_ERROR               = 0x82,
 587         NVME_SC_APPLICATION_TAG_CHECK_ERROR     = 0x83,
 588         NVME_SC_REFERENCE_TAG_CHECK_ERROR       = 0x84,
 589         NVME_SC_COMPARE_FAILURE                 = 0x85,
 590         NVME_SC_ACCESS_DENIED                   = 0x86,
 591         NVME_SC_DEALLOCATED_OR_UNWRITTEN        = 0x87,
 592 };
 593
 594 /* admin opcodes */
 595 enum nvme_admin_opcode {
 596         NVME_OPC_DELETE_IO_SQ                   = 0x00,
 597         NVME_OPC_CREATE_IO_SQ                   = 0x01,
 598         NVME_OPC_GET_LOG_PAGE                   = 0x02,
 599         /* 0x03 - reserved */
 600         NVME_OPC_DELETE_IO_CQ                   = 0x04,
 601         NVME_OPC_CREATE_IO_CQ                   = 0x05,
 602         NVME_OPC_IDENTIFY                       = 0x06,
 603         /* 0x07 - reserved */
 604         NVME_OPC_ABORT                          = 0x08,
 605         NVME_OPC_SET_FEATURES                   = 0x09,
 606         NVME_OPC_GET_FEATURES                   = 0x0a,
 607         /* 0x0b - reserved */
 608         NVME_OPC_ASYNC_EVENT_REQUEST            = 0x0c,
 609         NVME_OPC_NAMESPACE_MANAGEMENT           = 0x0d,
 610         /* 0x0e-0x0f - reserved */
 611         NVME_OPC_FIRMWARE_ACTIVATE              = 0x10,
 612         NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD        = 0x11,
 613         NVME_OPC_DEVICE_SELF_TEST               = 0x14,
 614         NVME_OPC_NAMESPACE_ATTACHMENT           = 0x15,
 615         NVME_OPC_KEEP_ALIVE                     = 0x18,
 616         NVME_OPC_DIRECTIVE_SEND                 = 0x19,
 617         NVME_OPC_DIRECTIVE_RECEIVE              = 0x1a,
 618         NVME_OPC_VIRTUALIZATION_MANAGEMENT      = 0x1c,
 619         NVME_OPC_NVME_MI_SEND                   = 0x1d,
 620         NVME_OPC_NVME_MI_RECEIVE                = 0x1e,
 621         NVME_OPC_DOORBELL_BUFFER_CONFIG         = 0x7c,
 622
 623         NVME_OPC_FORMAT_NVM                     = 0x80,
 624         NVME_OPC_SECURITY_SEND                  = 0x81,
 625         NVME_OPC_SECURITY_RECEIVE               = 0x82,
 626         NVME_OPC_SANITIZE                       = 0x84,
 627 };
 628
 629 /* nvme nvm opcodes */
 630 enum nvme_nvm_opcode {
 631         NVME_OPC_FLUSH                          = 0x00,
 632         NVME_OPC_WRITE                          = 0x01,
 633         NVME_OPC_READ                           = 0x02,
 634         /* 0x03 - reserved */
 635         NVME_OPC_WRITE_UNCORRECTABLE            = 0x04,
 636         NVME_OPC_COMPARE                        = 0x05,
 637         /* 0x06 - reserved */
 638         NVME_OPC_WRITE_ZEROES                   = 0x08,
 639         /* 0x07 - reserved */
 640         NVME_OPC_DATASET_MANAGEMENT             = 0x09,
 641         /* 0x0a-0x0c - reserved */
 642         NVME_OPC_RESERVATION_REGISTER           = 0x0d,
 643         NVME_OPC_RESERVATION_REPORT             = 0x0e,
 644         /* 0x0f-0x10 - reserved */
 645         NVME_OPC_RESERVATION_ACQUIRE            = 0x11,
 646         /* 0x12-0x14 - reserved */
 647         NVME_OPC_RESERVATION_RELEASE            = 0x15,
 648 };
 649
 650 enum nvme_feature {
 651         /* 0x00 - reserved */
 652         NVME_FEAT_ARBITRATION                   = 0x01,
 653         NVME_FEAT_POWER_MANAGEMENT              = 0x02,
 654         NVME_FEAT_LBA_RANGE_TYPE                = 0x03,
 655         NVME_FEAT_TEMPERATURE_THRESHOLD         = 0x04,
 656         NVME_FEAT_ERROR_RECOVERY                = 0x05,
 657         NVME_FEAT_VOLATILE_WRITE_CACHE          = 0x06,
 658         NVME_FEAT_NUMBER_OF_QUEUES              = 0x07,
 659         NVME_FEAT_INTERRUPT_COALESCING          = 0x08,
 660         NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION = 0x09,
 661         NVME_FEAT_WRITE_ATOMICITY               = 0x0A,
 662         NVME_FEAT_ASYNC_EVENT_CONFIGURATION     = 0x0B,
 663         NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION = 0x0C,
 664         NVME_FEAT_HOST_MEMORY_BUFFER            = 0x0D,
 665         NVME_FEAT_TIMESTAMP                     = 0x0E,
 666         NVME_FEAT_KEEP_ALIVE_TIMER              = 0x0F,
 667         NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT  = 0x10,
 668         NVME_FEAT_NON_OP_POWER_STATE_CONFIG     = 0x11,
 669         /* 0x12-0x77 - reserved */
 670         /* 0x78-0x7f - NVMe Management Interface */
 671         NVME_FEAT_SOFTWARE_PROGRESS_MARKER      = 0x80,
 672         /* 0x81-0xBF - command set specific (reserved) */
 673         /* 0xC0-0xFF - vendor specific */
 674 };
 675
 676 enum nvme_dsm_attribute {
 677         NVME_DSM_ATTR_INTEGRAL_READ             = 0x1,
 678         NVME_DSM_ATTR_INTEGRAL_WRITE            = 0x2,
 679         NVME_DSM_ATTR_DEALLOCATE                = 0x4,
 680 };
 681
 682 enum nvme_activate_action {
 683         NVME_AA_REPLACE_NO_ACTIVATE             = 0x0,
 684         NVME_AA_REPLACE_ACTIVATE                = 0x1,
 685         NVME_AA_ACTIVATE                        = 0x2,
 686 };
 687
 688 struct nvme_power_state {
 689         /** Maximum Power */
 690         uint16_t        mp;                     /* Maximum Power */
 691         uint8_t         ps_rsvd1;
 692         uint8_t         mps_nops;               /* Max Power Scale, Non-Operational State */
 693
 694         uint32_t        enlat;                  /* Entry Latency */
 695         uint32_t        exlat;                  /* Exit Latency */
 696
 697         uint8_t         rrt;                    /* Relative Read Throughput */
 698         uint8_t         rrl;                    /* Relative Read Latency */
 699         uint8_t         rwt;                    /* Relative Write Throughput */
 700         uint8_t         rwl;                    /* Relative Write Latency */
 701
 702         uint16_t        idlp;                   /* Idle Power */
 703         uint8_t         ips;                    /* Idle Power Scale */
 704         uint8_t         ps_rsvd8;
 705
 706         uint16_t        actp;                   /* Active Power */
 707         uint8_t         apw_aps;                /* Active Power Workload, Active Power Scale */
 708         uint8_t         ps_rsvd10[9];
 709 } __packed;
 710
 711 _Static_assert(sizeof(struct nvme_power_state) == 32, "bad size for nvme_power_state");
 712
 713 #define NVME_SERIAL_NUMBER_LENGTH       20
 714 #define NVME_MODEL_NUMBER_LENGTH        40
 715 #define NVME_FIRMWARE_REVISION_LENGTH   8
 716
 717 struct nvme_controller_data {
 718
 719         /* bytes 0-255: controller capabilities and features */
 720
 721         /** pci vendor id */
 722         uint16_t                vid;
 723
 724         /** pci subsystem vendor id */
 725         uint16_t                ssvid;
 726
 727         /** serial number */
 728         uint8_t                 sn[NVME_SERIAL_NUMBER_LENGTH];
 729
 730         /** model number */
 731         uint8_t                 mn[NVME_MODEL_NUMBER_LENGTH];
 732
 733         /** firmware revision */
 734         uint8_t                 fr[NVME_FIRMWARE_REVISION_LENGTH];
 735
 736         /** recommended arbitration burst */
 737         uint8_t                 rab;
 738
 739         /** ieee oui identifier */
 740         uint8_t                 ieee[3];
 741
 742         /** multi-interface capabilities */
 743         uint8_t                 mic;
 744
 745         /** maximum data transfer size */
 746         uint8_t                 mdts;
 747
 748         /** Controller ID */
 749         uint16_t                ctrlr_id;
 750
 751         /** Version */
 752         uint32_t                ver;
 753
 754         /** RTD3 Resume Latency */
 755         uint32_t                rtd3r;
 756
 757         /** RTD3 Enter Latency */
 758         uint32_t                rtd3e;
 759
 760         /** Optional Asynchronous Events Supported */
 761         uint32_t                oaes;   /* bitfield really */
 762
 763         /** Controller Attributes */
 764         uint32_t                ctratt; /* bitfield really */
 765
 766         uint8_t                 reserved1[12];
 767
 768         /** FRU Globally Unique Identifier */
 769         uint8_t                 fguid[16];
 770
 771         uint8_t                 reserved2[128];
 772
 773         /* bytes 256-511: admin command set attributes */
 774
 775         /** optional admin command support */
 776         uint16_t                oacs;
 777
 778         /** abort command limit */
 779         uint8_t                 acl;
 780
 781         /** asynchronous event request limit */
 782         uint8_t                 aerl;
 783
 784         /** firmware updates */
 785         uint8_t                 frmw;
 786
 787         /** log page attributes */
 788         uint8_t                 lpa;
 789
 790         /** error log page entries */
 791         uint8_t                 elpe;
 792
 793         /** number of power states supported */
 794         uint8_t                 npss;
 795
 796         /** admin vendor specific command configuration */
 797         uint8_t                 avscc;
 798
 799         /** Autonomous Power State Transition Attributes */
 800         uint8_t                 apsta;
 801
 802         /** Warning Composite Temperature Threshold */
 803         uint16_t                wctemp;
 804
 805         /** Critical Composite Temperature Threshold */
 806         uint16_t                cctemp;
 807
 808         /** Maximum Time for Firmware Activation */
 809         uint16_t                mtfa;
 810
 811         /** Host Memory Buffer Preferred Size */
 812         uint32_t                hmpre;
 813
 814         /** Host Memory Buffer Minimum Size */
 815         uint32_t                hmmin;
 816
 817         /** Name space capabilities  */
 818         struct {
 819                 /* if nsmgmt, report tnvmcap and unvmcap */
 820                 uint8_t    tnvmcap[16];
 821                 uint8_t    unvmcap[16];
 822         } __packed untncap;
 823
 824         /** Replay Protected Memory Block Support */
 825         uint32_t                rpmbs; /* Really a bitfield */
 826
 827         /** Extended Device Self-test Time */
 828         uint16_t                edstt;
 829
 830         /** Device Self-test Options */
 831         uint8_t                 dsto; /* Really a bitfield */
 832
 833         /** Firmware Update Granularity */
 834         uint8_t                 fwug;
 835
 836         /** Keep Alive Support */
 837         uint16_t                kas;
 838
 839         /** Host Controlled Thermal Management Attributes */
 840         uint16_t                hctma; /* Really a bitfield */
 841
 842         /** Minimum Thermal Management Temperature */
 843         uint16_t                mntmt;
 844
 845         /** Maximum Thermal Management Temperature */
 846         uint16_t                mxtmt;
 847
 848         /** Sanitize Capabilities */
 849         uint32_t                sanicap; /* Really a bitfield */
 850
 851         uint8_t                 reserved3[180];
 852         /* bytes 512-703: nvm command set attributes */
 853
 854         /** submission queue entry size */
 855         uint8_t                 sqes;
 856
 857         /** completion queue entry size */
 858         uint8_t                 cqes;
 859
 860         /** Maximum Outstanding Commands */
 861         uint16_t                maxcmd;
 862
 863         /** number of namespaces */
 864         uint32_t                nn;
 865
 866         /** optional nvm command support */
 867         uint16_t                oncs;
 868
 869         /** fused operation support */
 870         uint16_t                fuses;
 871
 872         /** format nvm attributes */
 873         uint8_t                 fna;
 874
 875         /** volatile write cache */
 876         uint8_t                 vwc;
 877
 878         /** Atomic Write Unit Normal */
 879         uint16_t                awun;
 880
 881         /** Atomic Write Unit Power Fail */
 882         uint16_t                awupf;
 883
 884         /** NVM Vendor Specific Command Configuration */
 885         uint8_t                 nvscc;
 886         uint8_t                 reserved5;
 887
 888         /** Atomic Compare & Write Unit */
 889         uint16_t                acwu;
 890         uint16_t                reserved6;
 891
 892         /** SGL Support */
 893         uint32_t                sgls;
 894
 895         /* bytes 540-767: Reserved */
 896         uint8_t                 reserved7[228];
 897
 898         /** NVM Subsystem NVMe Qualified Name */
 899         uint8_t                 subnqn[256];
 900
 901         /* bytes 1024-1791: Reserved */
 902         uint8_t                 reserved8[768];
 903
 904         /* bytes 1792-2047: NVMe over Fabrics specification */
 905         uint8_t                 reserved9[256];
 906
 907         /* bytes 2048-3071: power state descriptors */
 908         struct nvme_power_state power_state[32];
 909
 910         /* bytes 3072-4095: vendor specific */
 911         uint8_t                 vs[1024];
 912 } __packed __aligned(4);
 913
 914 _Static_assert(sizeof(struct nvme_controller_data) == 4096, "bad size for nvme_controller_data");
 915
 916 struct nvme_namespace_data {
 917
 918         /** namespace size */
 919         uint64_t                nsze;
 920
 921         /** namespace capacity */
 922         uint64_t                ncap;
 923
 924         /** namespace utilization */
 925         uint64_t                nuse;
 926
 927         /** namespace features */
 928         uint8_t                 nsfeat;
 929
 930         /** number of lba formats */
 931         uint8_t                 nlbaf;
 932
 933         /** formatted lba size */
 934         uint8_t                 flbas;
 935
 936         /** metadata capabilities */
 937         uint8_t                 mc;
 938
 939         /** end-to-end data protection capabilities */
 940         uint8_t                 dpc;
 941
 942         /** end-to-end data protection type settings */
 943         uint8_t                 dps;
 944
 945         /** Namespace Multi-path I/O and Namespace Sharing Capabilities */
 946         uint8_t                 nmic;
 947
 948         /** Reservation Capabilities */
 949         uint8_t                 rescap;
 950
 951         /** Format Progress Indicator */
 952         uint8_t                 fpi;
 953
 954         /** Deallocate Logical Block Features */
 955         uint8_t                 dlfeat;
 956
 957         /** Namespace Atomic Write Unit Normal  */
 958         uint16_t                nawun;
 959
 960         /** Namespace Atomic Write Unit Power Fail */
 961         uint16_t                nawupf;
 962
 963         /** Namespace Atomic Compare & Write Unit */
 964         uint16_t                nacwu;
 965
 966         /** Namespace Atomic Boundary Size Normal */
 967         uint16_t                nabsn;
 968
 969         /** Namespace Atomic Boundary Offset */
 970         uint16_t                nabo;
 971
 972         /** Namespace Atomic Boundary Size Power Fail */
 973         uint16_t                nabspf;
 974
 975         /** Namespace Optimal IO Boundary */
 976         uint16_t                noiob;
 977
 978         /** NVM Capacity */
 979         uint8_t                 nvmcap[16];
 980
 981         /* bytes 64-103: Reserved */
 982         uint8_t                 reserved5[40];
 983
 984         /** Namespace Globally Unique Identifier */
 985         uint8_t                 nguid[16];
 986
 987         /** IEEE Extended Unique Identifier */
 988         uint8_t                 eui64[8];
 989
 990         /** lba format support */
 991         uint32_t                lbaf[16];
 992
 993         uint8_t                 reserved6[192];
 994
 995         uint8_t                 vendor_specific[3712];
 996 } __packed __aligned(4);
 997
 998 _Static_assert(sizeof(struct nvme_namespace_data) == 4096, "bad size for nvme_namepsace_data");
 999
1000 enum nvme_log_page {
1001
1002         /* 0x00 - reserved */
1003         NVME_LOG_ERROR                  = 0x01,
1004         NVME_LOG_HEALTH_INFORMATION     = 0x02,
1005         NVME_LOG_FIRMWARE_SLOT          = 0x03,
1006         NVME_LOG_CHANGED_NAMESPACE      = 0x04,
1007         NVME_LOG_COMMAND_EFFECT         = 0x05,
1008         /* 0x06-0x7F - reserved */
1009         /* 0x80-0xBF - I/O command set specific */
1010         NVME_LOG_RES_NOTIFICATION       = 0x80,
1011         /* 0xC0-0xFF - vendor specific */
1012
1013         /*
1014          * The following are Intel Specific log pages, but they seem
1015          * to be widely implemented.
1016          */
1017         INTEL_LOG_READ_LAT_LOG          = 0xc1,
1018         INTEL_LOG_WRITE_LAT_LOG         = 0xc2,
1019         INTEL_LOG_TEMP_STATS            = 0xc5,
1020         INTEL_LOG_ADD_SMART             = 0xca,
1021         INTEL_LOG_DRIVE_MKT_NAME        = 0xdd,
1022
1023         /*
1024          * HGST log page, with lots ofs sub pages.
1025          */
1026         HGST_INFO_LOG                   = 0xc1,
1027 };
1028
1029 struct nvme_error_information_entry {
1030
1031         uint64_t                error_count;
1032         uint16_t                sqid;
1033         uint16_t                cid;
1034         uint16_t                status;
1035         uint16_t                error_location;
1036         uint64_t                lba;
1037         uint32_t                nsid;
1038         uint8_t                 vendor_specific;
1039         uint8_t                 reserved[35];
1040 } __packed __aligned(4);
1041
1042 _Static_assert(sizeof(struct nvme_error_information_entry) == 64, "bad size for nvme_error_information_entry");
1043
1044 struct nvme_health_information_page {
1045
1046         uint8_t                 critical_warning;
1047         uint16_t                temperature;
1048         uint8_t                 available_spare;
1049         uint8_t                 available_spare_threshold;
1050         uint8_t                 percentage_used;
1051
1052         uint8_t                 reserved[26];
1053
1054         /*
1055          * Note that the following are 128-bit values, but are
1056          *  defined as an array of 2 64-bit values.
1057          */
1058         /* Data Units Read is always in 512-byte units. */
1059         uint64_t                data_units_read[2];
1060         /* Data Units Written is always in 512-byte units. */
1061         uint64_t                data_units_written[2];
1062         /* For NVM command set, this includes Compare commands. */
1063         uint64_t                host_read_commands[2];
1064         uint64_t                host_write_commands[2];
1065         /* Controller Busy Time is reported in minutes. */
1066         uint64_t                controller_busy_time[2];
1067         uint64_t                power_cycles[2];
1068         uint64_t                power_on_hours[2];
1069         uint64_t                unsafe_shutdowns[2];
1070         uint64_t                media_errors[2];
1071         uint64_t                num_error_info_log_entries[2];
1072         uint32_t                warning_temp_time;
1073         uint32_t                error_temp_time;
1074         uint16_t                temp_sensor[8];
1075
1076         uint8_t                 reserved2[296];
1077 } __packed __aligned(4);
1078
1079 _Static_assert(sizeof(struct nvme_health_information_page) == 512, "bad size for nvme_health_information_page");
1080
1081 struct nvme_firmware_page {
1082
1083         uint8_t                 afi;
1084         uint8_t                 reserved[7];
1085         uint64_t                revision[7]; /* revisions for 7 slots */
1086         uint8_t                 reserved2[448];
1087 } __packed __aligned(4);
1088
1089 _Static_assert(sizeof(struct nvme_firmware_page) == 512, "bad size for nvme_firmware_page");
1090
1091 struct nvme_ns_list {
1092         uint32_t                ns[1024];
1093 } __packed __aligned(4);
1094
1095 _Static_assert(sizeof(struct nvme_ns_list) == 4096, "bad size for nvme_ns_list");
1096
1097 struct intel_log_temp_stats
1098 {
1099         uint64_t        current;
1100         uint64_t        overtemp_flag_last;
1101         uint64_t        overtemp_flag_life;
1102         uint64_t        max_temp;
1103         uint64_t        min_temp;
1104         uint64_t        _rsvd[5];
1105         uint64_t        max_oper_temp;
1106         uint64_t        min_oper_temp;
1107         uint64_t        est_offset;
1108 } __packed __aligned(4);
1109
1110 _Static_assert(sizeof(struct intel_log_temp_stats) == 13 * 8, "bad size for intel_log_temp_stats");
1111
1112 #define NVME_TEST_MAX_THREADS   128
1113
1114 struct nvme_io_test {
1115
1116         enum nvme_nvm_opcode    opc;
1117         uint32_t                size;
1118         uint32_t                time;   /* in seconds */
1119         uint32_t                num_threads;
1120         uint32_t                flags;
1121         uint64_t                io_completed[NVME_TEST_MAX_THREADS];
1122 };
1123
1124 enum nvme_io_test_flags {
1125
1126         /*
1127          * Specifies whether dev_refthread/dev_relthread should be
1128          *  called during NVME_BIO_TEST.  Ignored for other test
1129          *  types.
1130          */
1131         NVME_TEST_FLAG_REFTHREAD =      0x1,
1132 };
1133
1134 struct nvme_pt_command {
1135
1136         /*
1137          * cmd is used to specify a passthrough command to a controller or
1138          *  namespace.
1139          *
1140          * The following fields from cmd may be specified by the caller:
1141          *      * opc  (opcode)
1142          *      * nsid (namespace id) - for admin commands only
1143          *      * cdw10-cdw15
1144          *
1145          * Remaining fields must be set to 0 by the caller.
1146          */
1147         struct nvme_command     cmd;
1148
1149         /*
1150          * cpl returns completion status for the passthrough command
1151          *  specified by cmd.
1152          *
1153          * The following fields will be filled out by the driver, for
1154          *  consumption by the caller:
1155          *      * cdw0
1156          *      * status (except for phase)
1157          *
1158          * Remaining fields will be set to 0 by the driver.
1159          */
1160         struct nvme_completion  cpl;
1161
1162         /* buf is the data buffer associated with this passthrough command. */
1163         void *                  buf;
1164
1165         /*
1166          * len is the length of the data buffer associated with this
1167          *  passthrough command.
1168          */
1169         uint32_t                len;
1170
1171         /*
1172          * is_read = 1 if the passthrough command will read data into the
1173          *  supplied buffer from the controller.
1174          *
1175          * is_read = 0 if the passthrough command will write data from the
1176          *  supplied buffer to the controller.
1177          */
1178         uint32_t                is_read;
1179
1180         /*
1181          * driver_lock is used by the driver only.  It must be set to 0
1182          *  by the caller.
1183          */
1184         struct mtx *            driver_lock;
1185 };
1186
1187 #define nvme_completion_is_error(cpl)                                   \
1188         (NVME_STATUS_GET_SC((cpl)->status) != 0 || NVME_STATUS_GET_SCT((cpl)->status) != 0)
1189
1190 void    nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen);
1191
1192 #ifdef _KERNEL
1193
1194 struct bio;
1195
1196 struct nvme_namespace;
1197 struct nvme_controller;
1198 struct nvme_consumer;
1199
1200 typedef void (*nvme_cb_fn_t)(void *, const struct nvme_completion *);
1201
1202 typedef void *(*nvme_cons_ns_fn_t)(struct nvme_namespace *, void *);
1203 typedef void *(*nvme_cons_ctrlr_fn_t)(struct nvme_controller *);
1204 typedef void (*nvme_cons_async_fn_t)(void *, const struct nvme_completion *,
1205                                      uint32_t, void *, uint32_t);
1206 typedef void (*nvme_cons_fail_fn_t)(void *);
1207
1208 enum nvme_namespace_flags {
1209         NVME_NS_DEALLOCATE_SUPPORTED    = 0x1,
1210         NVME_NS_FLUSH_SUPPORTED         = 0x2,
1211 };
1212
1213 int     nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
1214                                    struct nvme_pt_command *pt,
1215                                    uint32_t nsid, int is_user_buffer,
1216                                    int is_admin_cmd);
1217
1218 /* Admin functions */
1219 void    nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr,
1220                                    uint8_t feature, uint32_t cdw11,
1221                                    void *payload, uint32_t payload_size,
1222                                    nvme_cb_fn_t cb_fn, void *cb_arg);
1223 void    nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr,
1224                                    uint8_t feature, uint32_t cdw11,
1225                                    void *payload, uint32_t payload_size,
1226                                    nvme_cb_fn_t cb_fn, void *cb_arg);
1227 void    nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr,
1228                                     uint8_t log_page, uint32_t nsid,
1229                                     void *payload, uint32_t payload_size,
1230                                     nvme_cb_fn_t cb_fn, void *cb_arg);
1231
1232 /* NVM I/O functions */
1233 int     nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload,
1234                           uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
1235                           void *cb_arg);
1236 int     nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp,
1237                               nvme_cb_fn_t cb_fn, void *cb_arg);
1238 int     nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload,
1239                          uint64_t lba, uint32_t lba_count, nvme_cb_fn_t cb_fn,
1240                          void *cb_arg);
1241 int     nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp,
1242                               nvme_cb_fn_t cb_fn, void *cb_arg);
1243 int     nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
1244                                uint8_t num_ranges, nvme_cb_fn_t cb_fn,
1245                                void *cb_arg);
1246 int     nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn,
1247                           void *cb_arg);
1248 int     nvme_ns_dump(struct nvme_namespace *ns, void *virt, off_t offset,
1249                      size_t len);
1250
1251 /* Registration functions */
1252 struct nvme_consumer *  nvme_register_consumer(nvme_cons_ns_fn_t    ns_fn,
1253                                                nvme_cons_ctrlr_fn_t ctrlr_fn,
1254                                                nvme_cons_async_fn_t async_fn,
1255                                                nvme_cons_fail_fn_t  fail_fn);
1256 void            nvme_unregister_consumer(struct nvme_consumer *consumer);
1257
1258 /* Controller helper functions */
1259 device_t        nvme_ctrlr_get_device(struct nvme_controller *ctrlr);
1260 const struct nvme_controller_data *
1261                 nvme_ctrlr_get_data(struct nvme_controller *ctrlr);
1262
1263 /* Namespace helper functions */
1264 uint32_t        nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns);
1265 uint32_t        nvme_ns_get_sector_size(struct nvme_namespace *ns);
1266 uint64_t        nvme_ns_get_num_sectors(struct nvme_namespace *ns);
1267 uint64_t        nvme_ns_get_size(struct nvme_namespace *ns);
1268 uint32_t        nvme_ns_get_flags(struct nvme_namespace *ns);
1269 const char *    nvme_ns_get_serial_number(struct nvme_namespace *ns);
1270 const char *    nvme_ns_get_model_number(struct nvme_namespace *ns);
1271 const struct nvme_namespace_data *
1272                 nvme_ns_get_data(struct nvme_namespace *ns);
1273 uint32_t        nvme_ns_get_stripesize(struct nvme_namespace *ns);
1274
1275 int     nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp,
1276                             nvme_cb_fn_t cb_fn);
1277
1278 /*
1279  * Command building helper functions -- shared with CAM
1280  * These functions assume allocator zeros out cmd structure
1281  * CAM's xpt_get_ccb and the request allocator for nvme both
1282  * do zero'd allocations.
1283  */
1284 static inline
1285 void    nvme_ns_flush_cmd(struct nvme_command *cmd, uint32_t nsid)
1286 {
1287
1288         cmd->opc = NVME_OPC_FLUSH;
1289         cmd->nsid = htole32(nsid);
1290 }
1291
1292 static inline
1293 void    nvme_ns_rw_cmd(struct nvme_command *cmd, uint32_t rwcmd, uint32_t nsid,
1294     uint64_t lba, uint32_t count)
1295 {
1296         cmd->opc = rwcmd;
1297         cmd->nsid = htole32(nsid);
1298         cmd->cdw10 = htole32(lba & 0xffffffffu);
1299         cmd->cdw11 = htole32(lba >> 32);
1300         cmd->cdw12 = htole32(count-1);
1301 }
1302
1303 static inline
1304 void    nvme_ns_write_cmd(struct nvme_command *cmd, uint32_t nsid,
1305     uint64_t lba, uint32_t count)
1306 {
1307         nvme_ns_rw_cmd(cmd, NVME_OPC_WRITE, nsid, lba, count);
1308 }
1309
1310 static inline
1311 void    nvme_ns_read_cmd(struct nvme_command *cmd, uint32_t nsid,
1312     uint64_t lba, uint32_t count)
1313 {
1314         nvme_ns_rw_cmd(cmd, NVME_OPC_READ, nsid, lba, count);
1315 }
1316
1317 static inline
1318 void    nvme_ns_trim_cmd(struct nvme_command *cmd, uint32_t nsid,
1319     uint32_t num_ranges)
1320 {
1321         cmd->opc = NVME_OPC_DATASET_MANAGEMENT;
1322         cmd->nsid = htole32(nsid);
1323         cmd->cdw10 = htole32(num_ranges - 1);
1324         cmd->cdw11 = htole32(NVME_DSM_ATTR_DEALLOCATE);
1325 }
1326
1327 extern int nvme_use_nvd;
1328
1329 #endif /* _KERNEL */
1330
1331 /* Endianess conversion functions for NVMe structs */
1332 static inline
1333 void    nvme_completion_swapbytes(struct nvme_completion *s)
1334 {
1335
1336         s->cdw0 = le32toh(s->cdw0);
1337         /* omit rsvd1 */
1338         s->sqhd = le16toh(s->sqhd);
1339         s->sqid = le16toh(s->sqid);
1340         /* omit cid */
1341         s->status = le16toh(s->status);
1342 }
1343
1344 static inline
1345 void    nvme_power_state_swapbytes(struct nvme_power_state *s)
1346 {
1347
1348         s->mp = le16toh(s->mp);
1349         s->enlat = le32toh(s->enlat);
1350         s->exlat = le32toh(s->exlat);
1351         s->idlp = le16toh(s->idlp);
1352         s->actp = le16toh(s->actp);
1353 }
1354
1355 static inline
1356 void    nvme_controller_data_swapbytes(struct nvme_controller_data *s)
1357 {
1358         int i;
1359
1360         s->vid = le16toh(s->vid);
1361         s->ssvid = le16toh(s->ssvid);
1362         s->ctrlr_id = le16toh(s->ctrlr_id);
1363         s->ver = le32toh(s->ver);
1364         s->rtd3r = le32toh(s->rtd3r);
1365         s->rtd3e = le32toh(s->rtd3e);
1366         s->oaes = le32toh(s->oaes);
1367         s->ctratt = le32toh(s->ctratt);
1368         s->oacs = le16toh(s->oacs);
1369         s->wctemp = le16toh(s->wctemp);
1370         s->cctemp = le16toh(s->cctemp);
1371         s->mtfa = le16toh(s->mtfa);
1372         s->hmpre = le32toh(s->hmpre);
1373         s->hmmin = le32toh(s->hmmin);
1374         s->rpmbs = le32toh(s->rpmbs);
1375         s->edstt = le16toh(s->edstt);
1376         s->kas = le16toh(s->kas);
1377         s->hctma = le16toh(s->hctma);
1378         s->mntmt = le16toh(s->mntmt);
1379         s->mxtmt = le16toh(s->mxtmt);
1380         s->sanicap = le32toh(s->sanicap);
1381         s->maxcmd = le16toh(s->maxcmd);
1382         s->nn = le32toh(s->nn);
1383         s->oncs = le16toh(s->oncs);
1384         s->fuses = le16toh(s->fuses);
1385         s->awun = le16toh(s->awun);
1386         s->awupf = le16toh(s->awupf);
1387         s->acwu = le16toh(s->acwu);
1388         s->sgls = le32toh(s->sgls);
1389         for (i = 0; i < 32; i++)
1390                 nvme_power_state_swapbytes(&s->power_state[i]);
1391 }
1392
1393 static inline
1394 void    nvme_namespace_data_swapbytes(struct nvme_namespace_data *s)
1395 {
1396         int i;
1397
1398         s->nsze = le64toh(s->nsze);
1399         s->ncap = le64toh(s->ncap);
1400         s->nuse = le64toh(s->nuse);
1401         s->nawun = le16toh(s->nawun);
1402         s->nawupf = le16toh(s->nawupf);
1403         s->nacwu = le16toh(s->nacwu);
1404         s->nabsn = le16toh(s->nabsn);
1405         s->nabo = le16toh(s->nabo);
1406         s->nabspf = le16toh(s->nabspf);
1407         s->noiob = le16toh(s->noiob);
1408         for (i = 0; i < 16; i++)
1409                 s->lbaf[i] = le32toh(s->lbaf[i]);
1410 }
1411
1412 static inline
1413 void    nvme_error_information_entry_swapbytes(struct nvme_error_information_entry *s)
1414 {
1415
1416         s->error_count = le64toh(s->error_count);
1417         s->sqid = le16toh(s->sqid);
1418         s->cid = le16toh(s->cid);
1419         s->status = le16toh(s->status);
1420         s->error_location = le16toh(s->error_location);
1421         s->lba = le64toh(s->lba);
1422         s->nsid = le32toh(s->nsid);
1423 }
1424
1425 static inline
1426 void    nvme_le128toh(void *p)
1427 {
1428 #if _BYTE_ORDER != _LITTLE_ENDIAN
1429         /* Swap 16 bytes in place */
1430         char *tmp = (char*)p;
1431         char b;
1432         int i;
1433         for (i = 0; i < 8; i++) {
1434                 b = tmp[i];
1435                 tmp[i] = tmp[15-i];
1436                 tmp[15-i] = b;
1437         }
1438 #else
1439         (void)p;
1440 #endif
1441 }
1442
1443 static inline
1444 void    nvme_health_information_page_swapbytes(struct nvme_health_information_page *s)
1445 {
1446         int i;
1447
1448         s->temperature = le16toh(s->temperature);
1449         nvme_le128toh((void *)s->data_units_read);
1450         nvme_le128toh((void *)s->data_units_written);
1451         nvme_le128toh((void *)s->host_read_commands);
1452         nvme_le128toh((void *)s->host_write_commands);
1453         nvme_le128toh((void *)s->controller_busy_time);
1454         nvme_le128toh((void *)s->power_cycles);
1455         nvme_le128toh((void *)s->power_on_hours);
1456         nvme_le128toh((void *)s->unsafe_shutdowns);
1457         nvme_le128toh((void *)s->media_errors);
1458         nvme_le128toh((void *)s->num_error_info_log_entries);
1459         s->warning_temp_time = le32toh(s->warning_temp_time);
1460         s->error_temp_time = le32toh(s->error_temp_time);
1461         for (i = 0; i < 8; i++)
1462                 s->temp_sensor[i] = le16toh(s->temp_sensor[i]);
1463 }
1464
1465
1466 static inline
1467 void    nvme_firmware_page_swapbytes(struct nvme_firmware_page *s)
1468 {
1469         int i;
1470
1471         for (i = 0; i < 7; i++)
1472                 s->revision[i] = le64toh(s->revision[i]);
1473 }
1474
1475 static inline
1476 void    nvme_ns_list_swapbytes(struct nvme_ns_list *s)
1477 {
1478         int i;
1479
1480         for (i = 0; i < 1024; i++)
1481                 s->ns[i] = le32toh(s->ns[i]);
1482 }
1483
1484 static inline
1485 void    intel_log_temp_stats_swapbytes(struct intel_log_temp_stats *s)
1486 {
1487
1488         s->current = le64toh(s->current);
1489         s->overtemp_flag_last = le64toh(s->overtemp_flag_last);
1490         s->overtemp_flag_life = le64toh(s->overtemp_flag_life);
1491         s->max_temp = le64toh(s->max_temp);
1492         s->min_temp = le64toh(s->min_temp);
1493         /* omit _rsvd[] */
1494         s->max_oper_temp = le64toh(s->max_oper_temp);
1495         s->min_oper_temp = le64toh(s->min_oper_temp);
1496         s->est_offset = le64toh(s->est_offset);
1497 }
1498
1499 #endif /* __NVME_H__ */