usr.sbin/bhyve/pci_nvme.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2017 Shunsuke Mie
   5  * Copyright (c) 2018 Leon Dang
   6  * Copyright (c) 2020 Chuck Tuffli
   7  *
   8  * Function crc16 Copyright (c) 2017, Fedor Uporov
   9  *     Obtained from function ext2_crc16() in sys/fs/ext2fs/ext2_csum.c
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  30  * SUCH DAMAGE.
  31  */
  32
  33 /*
  34  * bhyve PCIe-NVMe device emulation.
  35  *
  36  * options:
  37  *  -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=#,dsm=<opt>
  38  *
  39  *  accepted devpath:
  40  *    /dev/blockdev
  41  *    /path/to/image
  42  *    ram=size_in_MiB
  43  *
  44  *  maxq    = max number of queues
  45  *  qsz     = max elements in each queue
  46  *  ioslots = max number of concurrent io requests
  47  *  sectsz  = sector size (defaults to blockif sector size)
  48  *  ser     = serial number (20-chars max)
  49  *  eui64   = IEEE Extended Unique Identifier (8 byte value)
  50  *  dsm     = DataSet Management support. Option is one of auto, enable,disable
  51  *
  52  */
  53
  54 /* TODO:
  55     - create async event for smart and log
  56     - intr coalesce
  57  */
  58
  59 #include <sys/cdefs.h>
  60 __FBSDID("$FreeBSD$");
  61
  62 #include <sys/errno.h>
  63 #include <sys/types.h>
  64 #include <net/ieee_oui.h>
  65
  66 #include <assert.h>
  67 #include <pthread.h>
  68 #include <pthread_np.h>
  69 #include <semaphore.h>
  70 #include <stdbool.h>
  71 #include <stddef.h>
  72 #include <stdint.h>
  73 #include <stdio.h>
  74 #include <stdlib.h>
  75 #include <string.h>
  76
  77 #include <machine/atomic.h>
  78 #include <machine/vmm.h>
  79 #include <vmmapi.h>
  80
  81 #include <dev/nvme/nvme.h>
  82
  83 #include "bhyverun.h"
  84 #include "block_if.h"
  85 #include "config.h"
  86 #include "debug.h"
  87 #include "pci_emul.h"
  88
  89
  90 static int nvme_debug = 0;
  91 #define DPRINTF(fmt, args...) if (nvme_debug) PRINTLN(fmt, ##args)
  92 #define WPRINTF(fmt, args...) PRINTLN(fmt, ##args)
  93
  94 /* defaults; can be overridden */
  95 #define NVME_MSIX_BAR           4
  96
  97 #define NVME_IOSLOTS            8
  98
  99 /* The NVMe spec defines bits 13:4 in BAR0 as reserved */
 100 #define NVME_MMIO_SPACE_MIN     (1 << 14)
 101
 102 #define NVME_QUEUES             16
 103 #define NVME_MAX_QENTRIES       2048
 104 /* Memory Page size Minimum reported in CAP register */
 105 #define NVME_MPSMIN             0
 106 /* MPSMIN converted to bytes */
 107 #define NVME_MPSMIN_BYTES       (1 << (12 + NVME_MPSMIN))
 108
 109 #define NVME_PRP2_ITEMS         (PAGE_SIZE/sizeof(uint64_t))
 110 #define NVME_MDTS               9
 111 /* Note the + 1 allows for the initial descriptor to not be page aligned */
 112 #define NVME_MAX_IOVEC          ((1 << NVME_MDTS) + 1)
 113 #define NVME_MAX_DATA_SIZE      ((1 << NVME_MDTS) * NVME_MPSMIN_BYTES)
 114
 115 /* This is a synthetic status code to indicate there is no status */
 116 #define NVME_NO_STATUS          0xffff
 117 #define NVME_COMPLETION_VALID(c)        ((c).status != NVME_NO_STATUS)
 118
 119 /* Reported temperature in Kelvin (i.e. room temperature) */
 120 #define NVME_TEMPERATURE 296
 121
 122 /* helpers */
 123
 124 /* Convert a zero-based value into a one-based value */
 125 #define ONE_BASED(zero)         ((zero) + 1)
 126 /* Convert a one-based value into a zero-based value */
 127 #define ZERO_BASED(one)         ((one)  - 1)
 128
 129 /* Encode number of SQ's and CQ's for Set/Get Features */
 130 #define NVME_FEATURE_NUM_QUEUES(sc) \
 131         (ZERO_BASED((sc)->num_squeues) & 0xffff) | \
 132         (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16;
 133
 134 #define NVME_DOORBELL_OFFSET    offsetof(struct nvme_registers, doorbell)
 135
 136 enum nvme_controller_register_offsets {
 137         NVME_CR_CAP_LOW = 0x00,
 138         NVME_CR_CAP_HI  = 0x04,
 139         NVME_CR_VS      = 0x08,
 140         NVME_CR_INTMS   = 0x0c,
 141         NVME_CR_INTMC   = 0x10,
 142         NVME_CR_CC      = 0x14,
 143         NVME_CR_CSTS    = 0x1c,
 144         NVME_CR_NSSR    = 0x20,
 145         NVME_CR_AQA     = 0x24,
 146         NVME_CR_ASQ_LOW = 0x28,
 147         NVME_CR_ASQ_HI  = 0x2c,
 148         NVME_CR_ACQ_LOW = 0x30,
 149         NVME_CR_ACQ_HI  = 0x34,
 150 };
 151
 152 enum nvme_cmd_cdw11 {
 153         NVME_CMD_CDW11_PC  = 0x0001,
 154         NVME_CMD_CDW11_IEN = 0x0002,
 155         NVME_CMD_CDW11_IV  = 0xFFFF0000,
 156 };
 157
 158 enum nvme_copy_dir {
 159         NVME_COPY_TO_PRP,
 160         NVME_COPY_FROM_PRP,
 161 };
 162
 163 #define NVME_CQ_INTEN   0x01
 164 #define NVME_CQ_INTCOAL 0x02
 165
 166 struct nvme_completion_queue {
 167         struct nvme_completion *qbase;
 168         pthread_mutex_t mtx;
 169         uint32_t        size;
 170         uint16_t        tail; /* nvme progress */
 171         uint16_t        head; /* guest progress */
 172         uint16_t        intr_vec;
 173         uint32_t        intr_en;
 174 };
 175
 176 struct nvme_submission_queue {
 177         struct nvme_command *qbase;
 178         pthread_mutex_t mtx;
 179         uint32_t        size;
 180         uint16_t        head; /* nvme progress */
 181         uint16_t        tail; /* guest progress */
 182         uint16_t        cqid; /* completion queue id */
 183         int             qpriority;
 184 };
 185
 186 enum nvme_storage_type {
 187         NVME_STOR_BLOCKIF = 0,
 188         NVME_STOR_RAM = 1,
 189 };
 190
 191 struct pci_nvme_blockstore {
 192         enum nvme_storage_type type;
 193         void            *ctx;
 194         uint64_t        size;
 195         uint32_t        sectsz;
 196         uint32_t        sectsz_bits;
 197         uint64_t        eui64;
 198         uint32_t        deallocate:1;
 199 };
 200
 201 /*
 202  * Calculate the number of additional page descriptors for guest IO requests
 203  * based on the advertised Max Data Transfer (MDTS) and given the number of
 204  * default iovec's in a struct blockif_req.
 205  */
 206 #define MDTS_PAD_SIZE \
 207         ( NVME_MAX_IOVEC > BLOCKIF_IOV_MAX ? \
 208           NVME_MAX_IOVEC - BLOCKIF_IOV_MAX : \
 209           0 )
 210
 211 struct pci_nvme_ioreq {
 212         struct pci_nvme_softc *sc;
 213         STAILQ_ENTRY(pci_nvme_ioreq) link;
 214         struct nvme_submission_queue *nvme_sq;
 215         uint16_t        sqid;
 216
 217         /* command information */
 218         uint16_t        opc;
 219         uint16_t        cid;
 220         uint32_t        nsid;
 221
 222         uint64_t        prev_gpaddr;
 223         size_t          prev_size;
 224         size_t          bytes;
 225
 226         struct blockif_req io_req;
 227
 228         struct iovec    iovpadding[MDTS_PAD_SIZE];
 229 };
 230
 231 enum nvme_dsm_type {
 232         /* Dataset Management bit in ONCS reflects backing storage capability */
 233         NVME_DATASET_MANAGEMENT_AUTO,
 234         /* Unconditionally set Dataset Management bit in ONCS */
 235         NVME_DATASET_MANAGEMENT_ENABLE,
 236         /* Unconditionally clear Dataset Management bit in ONCS */
 237         NVME_DATASET_MANAGEMENT_DISABLE,
 238 };
 239
 240 struct pci_nvme_softc;
 241 struct nvme_feature_obj;
 242
 243 typedef void (*nvme_feature_cb)(struct pci_nvme_softc *,
 244     struct nvme_feature_obj *,
 245     struct nvme_command *,
 246     struct nvme_completion *);
 247
 248 struct nvme_feature_obj {
 249         uint32_t        cdw11;
 250         nvme_feature_cb set;
 251         nvme_feature_cb get;
 252         bool namespace_specific;
 253 };
 254
 255 #define NVME_FID_MAX            (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1)
 256
 257 typedef enum {
 258         PCI_NVME_AE_TYPE_ERROR = 0,
 259         PCI_NVME_AE_TYPE_SMART,
 260         PCI_NVME_AE_TYPE_NOTICE,
 261         PCI_NVME_AE_TYPE_IO_CMD = 6,
 262         PCI_NVME_AE_TYPE_VENDOR = 7,
 263         PCI_NVME_AE_TYPE_MAX            /* Must be last */
 264 } pci_nvme_async_type;
 265
 266 /* Asynchronous Event Requests */
 267 struct pci_nvme_aer {
 268         STAILQ_ENTRY(pci_nvme_aer) link;
 269         uint16_t        cid;    /* Command ID of the submitted AER */
 270 };
 271
 272 /** Asynchronous Event Information - Notice */
 273 typedef enum {
 274         PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED = 0,
 275         PCI_NVME_AEI_NOTICE_FW_ACTIVATION,
 276         PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE,
 277         PCI_NVME_AEI_NOTICE_ANA_CHANGE,
 278         PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE,
 279         PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT,
 280         PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE,
 281         PCI_NVME_AEI_NOTICE_MAX,
 282 } pci_nvme_async_event_info_notice;
 283
 284 #define PCI_NVME_AEI_NOTICE_SHIFT               8
 285 #define PCI_NVME_AEI_NOTICE_MASK(event) (1 << (event + PCI_NVME_AEI_NOTICE_SHIFT))
 286
 287 /* Asynchronous Event Notifications */
 288 struct pci_nvme_aen {
 289         pci_nvme_async_type atype;
 290         uint32_t        event_data;
 291         bool            posted;
 292 };
 293
 294 /*
 295  * By default, enable all Asynchrnous Event Notifications:
 296  *     SMART / Health Critical Warnings
 297  *     Namespace Attribute Notices
 298  */
 299 #define PCI_NVME_AEN_DEFAULT_MASK       0x11f
 300
 301 typedef enum {
 302         NVME_CNTRLTYPE_IO = 1,
 303         NVME_CNTRLTYPE_DISCOVERY = 2,
 304         NVME_CNTRLTYPE_ADMIN = 3,
 305 } pci_nvme_cntrl_type;
 306
 307 struct pci_nvme_softc {
 308         struct pci_devinst *nsc_pi;
 309
 310         pthread_mutex_t mtx;
 311
 312         struct nvme_registers regs;
 313
 314         struct nvme_namespace_data  nsdata;
 315         struct nvme_controller_data ctrldata;
 316         struct nvme_error_information_entry err_log;
 317         struct nvme_health_information_page health_log;
 318         struct nvme_firmware_page fw_log;
 319         struct nvme_ns_list ns_log;
 320
 321         struct pci_nvme_blockstore nvstore;
 322
 323         uint16_t        max_qentries;   /* max entries per queue */
 324         uint32_t        max_queues;     /* max number of IO SQ's or CQ's */
 325         uint32_t        num_cqueues;
 326         uint32_t        num_squeues;
 327         bool            num_q_is_set; /* Has host set Number of Queues */
 328
 329         struct pci_nvme_ioreq *ioreqs;
 330         STAILQ_HEAD(, pci_nvme_ioreq) ioreqs_free; /* free list of ioreqs */
 331         uint32_t        pending_ios;
 332         uint32_t        ioslots;
 333         sem_t           iosemlock;
 334
 335         /*
 336          * Memory mapped Submission and Completion queues
 337          * Each array includes both Admin and IO queues
 338          */
 339         struct nvme_completion_queue *compl_queues;
 340         struct nvme_submission_queue *submit_queues;
 341
 342         struct nvme_feature_obj feat[NVME_FID_MAX];
 343
 344         enum nvme_dsm_type dataset_management;
 345
 346         /* Accounting for SMART data */
 347         __uint128_t     read_data_units;
 348         __uint128_t     write_data_units;
 349         __uint128_t     read_commands;
 350         __uint128_t     write_commands;
 351         uint32_t        read_dunits_remainder;
 352         uint32_t        write_dunits_remainder;
 353
 354         STAILQ_HEAD(, pci_nvme_aer) aer_list;
 355         pthread_mutex_t aer_mtx;
 356         uint32_t        aer_count;
 357         struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX];
 358         pthread_t       aen_tid;
 359         pthread_mutex_t aen_mtx;
 360         pthread_cond_t  aen_cond;
 361 };
 362
 363
 364 static void pci_nvme_cq_update(struct pci_nvme_softc *sc,
 365     struct nvme_completion_queue *cq,
 366     uint32_t cdw0,
 367     uint16_t cid,
 368     uint16_t sqid,
 369     uint16_t status);
 370 static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *);
 371 static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *);
 372 static void pci_nvme_io_done(struct blockif_req *, int);
 373
 374 /* Controller Configuration utils */
 375 #define NVME_CC_GET_EN(cc) \
 376         ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK)
 377 #define NVME_CC_GET_CSS(cc) \
 378         ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK)
 379 #define NVME_CC_GET_SHN(cc) \
 380         ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK)
 381 #define NVME_CC_GET_IOSQES(cc) \
 382         ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK)
 383 #define NVME_CC_GET_IOCQES(cc) \
 384         ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK)
 385
 386 #define NVME_CC_WRITE_MASK \
 387         ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \
 388          (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \
 389          (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT))
 390
 391 #define NVME_CC_NEN_WRITE_MASK \
 392         ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \
 393          (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \
 394          (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT))
 395
 396 /* Controller Status utils */
 397 #define NVME_CSTS_GET_RDY(sts) \
 398         ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK)
 399
 400 #define NVME_CSTS_RDY   (1 << NVME_CSTS_REG_RDY_SHIFT)
 401
 402 /* Completion Queue status word utils */
 403 #define NVME_STATUS_P   (1 << NVME_STATUS_P_SHIFT)
 404 #define NVME_STATUS_MASK \
 405         ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\
 406          (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT))
 407
 408 #define NVME_ONCS_DSM   (NVME_CTRLR_DATA_ONCS_DSM_MASK << \
 409         NVME_CTRLR_DATA_ONCS_DSM_SHIFT)
 410
 411 static void nvme_feature_invalid_cb(struct pci_nvme_softc *,
 412     struct nvme_feature_obj *,
 413     struct nvme_command *,
 414     struct nvme_completion *);
 415 static void nvme_feature_temperature(struct pci_nvme_softc *,
 416     struct nvme_feature_obj *,
 417     struct nvme_command *,
 418     struct nvme_completion *);
 419 static void nvme_feature_num_queues(struct pci_nvme_softc *,
 420     struct nvme_feature_obj *,
 421     struct nvme_command *,
 422     struct nvme_completion *);
 423 static void nvme_feature_iv_config(struct pci_nvme_softc *,
 424     struct nvme_feature_obj *,
 425     struct nvme_command *,
 426     struct nvme_completion *);
 427 static void nvme_feature_async_event(struct pci_nvme_softc *,
 428     struct nvme_feature_obj *,
 429     struct nvme_command *,
 430     struct nvme_completion *);
 431
 432 static void *aen_thr(void *arg);
 433
 434 static __inline void
 435 cpywithpad(char *dst, size_t dst_size, const char *src, char pad)
 436 {
 437         size_t len;
 438
 439         len = strnlen(src, dst_size);
 440         memset(dst, pad, dst_size);
 441         memcpy(dst, src, len);
 442 }
 443
 444 static __inline void
 445 pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code)
 446 {
 447
 448         *status &= ~NVME_STATUS_MASK;
 449         *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT |
 450                 (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT;
 451 }
 452
 453 static __inline void
 454 pci_nvme_status_genc(uint16_t *status, uint16_t code)
 455 {
 456
 457         pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);
 458 }
 459
 460 /*
 461  * Initialize the requested number or IO Submission and Completion Queues.
 462  * Admin queues are allocated implicitly.
 463  */
 464 static void
 465 pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq)
 466 {
 467         uint32_t i;
 468
 469         /*
 470          * Allocate and initialize the Submission Queues
 471          */
 472         if (nsq > NVME_QUEUES) {
 473                 WPRINTF("%s: clamping number of SQ from %u to %u",
 474                                         __func__, nsq, NVME_QUEUES);
 475                 nsq = NVME_QUEUES;
 476         }
 477
 478         sc->num_squeues = nsq;
 479
 480         sc->submit_queues = calloc(sc->num_squeues + 1,
 481                                 sizeof(struct nvme_submission_queue));
 482         if (sc->submit_queues == NULL) {
 483                 WPRINTF("%s: SQ allocation failed", __func__);
 484                 sc->num_squeues = 0;
 485         } else {
 486                 struct nvme_submission_queue *sq = sc->submit_queues;
 487
 488                 for (i = 0; i < sc->num_squeues; i++)
 489                         pthread_mutex_init(&sq[i].mtx, NULL);
 490         }
 491
 492         /*
 493          * Allocate and initialize the Completion Queues
 494          */
 495         if (ncq > NVME_QUEUES) {
 496                 WPRINTF("%s: clamping number of CQ from %u to %u",
 497                                         __func__, ncq, NVME_QUEUES);
 498                 ncq = NVME_QUEUES;
 499         }
 500
 501         sc->num_cqueues = ncq;
 502
 503         sc->compl_queues = calloc(sc->num_cqueues + 1,
 504                                 sizeof(struct nvme_completion_queue));
 505         if (sc->compl_queues == NULL) {
 506                 WPRINTF("%s: CQ allocation failed", __func__);
 507                 sc->num_cqueues = 0;
 508         } else {
 509                 struct nvme_completion_queue *cq = sc->compl_queues;
 510
 511                 for (i = 0; i < sc->num_cqueues; i++)
 512                         pthread_mutex_init(&cq[i].mtx, NULL);
 513         }
 514 }
 515
 516 static void
 517 pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)
 518 {
 519         struct nvme_controller_data *cd = &sc->ctrldata;
 520
 521         cd->vid = 0xFB5D;
 522         cd->ssvid = 0x0000;
 523
 524         cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' ');
 525         cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' ');
 526
 527         /* Num of submission commands that we can handle at a time (2^rab) */
 528         cd->rab   = 4;
 529
 530         /* FreeBSD OUI */
 531         cd->ieee[0] = 0x58;
 532         cd->ieee[1] = 0x9c;
 533         cd->ieee[2] = 0xfc;
 534
 535         cd->mic = 0;
 536
 537         cd->mdts = NVME_MDTS;   /* max data transfer size (2^mdts * CAP.MPSMIN) */
 538
 539         cd->ver = NVME_REV(1,4);
 540
 541         cd->cntrltype = NVME_CNTRLTYPE_IO;
 542         cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT;
 543         cd->oaes = NVMEB(NVME_CTRLR_DATA_OAES_NS_ATTR);
 544         cd->acl = 2;
 545         cd->aerl = 4;
 546
 547         /* Advertise 1, Read-only firmware slot */
 548         cd->frmw = NVME_CTRLR_DATA_FRMW_SLOT1_RO_MASK |
 549             (1 << NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT);
 550         cd->lpa = 0;    /* TODO: support some simple things like SMART */
 551         cd->elpe = 0;   /* max error log page entries */
 552         cd->npss = 1;   /* number of power states support */
 553
 554         /* Warning Composite Temperature Threshold */
 555         cd->wctemp = 0x0157;
 556         cd->cctemp = 0x0157;
 557
 558         cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) |
 559             (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT);
 560         cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) |
 561             (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT);
 562         cd->nn = 1;     /* number of namespaces */
 563
 564         cd->oncs = 0;
 565         switch (sc->dataset_management) {
 566         case NVME_DATASET_MANAGEMENT_AUTO:
 567                 if (sc->nvstore.deallocate)
 568                         cd->oncs |= NVME_ONCS_DSM;
 569                 break;
 570         case NVME_DATASET_MANAGEMENT_ENABLE:
 571                 cd->oncs |= NVME_ONCS_DSM;
 572                 break;
 573         default:
 574                 break;
 575         }
 576
 577         cd->fna = NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK <<
 578             NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT;
 579
 580         cd->vwc = NVME_CTRLR_DATA_VWC_ALL_NO << NVME_CTRLR_DATA_VWC_ALL_SHIFT;
 581
 582         cd->power_state[0].mp = 10;
 583 }
 584
 585 /*
 586  * Calculate the CRC-16 of the given buffer
 587  * See copyright attribution at top of file
 588  */
 589 static uint16_t
 590 crc16(uint16_t crc, const void *buffer, unsigned int len)
 591 {
 592         const unsigned char *cp = buffer;
 593         /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */
 594         static uint16_t const crc16_table[256] = {
 595                 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
 596                 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
 597                 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
 598                 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
 599                 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
 600                 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
 601                 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
 602                 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
 603                 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
 604                 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
 605                 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
 606                 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
 607                 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
 608                 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
 609                 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
 610                 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
 611                 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
 612                 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
 613                 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
 614                 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
 615                 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
 616                 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
 617                 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
 618                 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
 619                 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
 620                 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
 621                 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
 622                 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
 623                 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
 624                 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
 625                 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
 626                 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040
 627         };
 628
 629         while (len--)
 630                 crc = (((crc >> 8) & 0xffU) ^
 631                     crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU;
 632         return crc;
 633 }
 634
 635 static void
 636 pci_nvme_init_nsdata_size(struct pci_nvme_blockstore *nvstore,
 637     struct nvme_namespace_data *nd)
 638 {
 639
 640         /* Get capacity and block size information from backing store */
 641         nd->nsze = nvstore->size / nvstore->sectsz;
 642         nd->ncap = nd->nsze;
 643         nd->nuse = nd->nsze;
 644 }
 645
 646 static void
 647 pci_nvme_init_nsdata(struct pci_nvme_softc *sc,
 648     struct nvme_namespace_data *nd, uint32_t nsid,
 649     struct pci_nvme_blockstore *nvstore)
 650 {
 651
 652         pci_nvme_init_nsdata_size(nvstore, nd);
 653
 654         if (nvstore->type == NVME_STOR_BLOCKIF)
 655                 nvstore->deallocate = blockif_candelete(nvstore->ctx);
 656
 657         nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */
 658         nd->flbas = 0;
 659
 660         /* Create an EUI-64 if user did not provide one */
 661         if (nvstore->eui64 == 0) {
 662                 char *data = NULL;
 663                 uint64_t eui64 = nvstore->eui64;
 664
 665                 asprintf(&data, "%s%u%u%u", get_config_value("name"),
 666                     sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot,
 667                     sc->nsc_pi->pi_func);
 668
 669                 if (data != NULL) {
 670                         eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data));
 671                         free(data);
 672                 }
 673                 nvstore->eui64 = (eui64 << 16) | (nsid & 0xffff);
 674         }
 675         be64enc(nd->eui64, nvstore->eui64);
 676
 677         /* LBA data-sz = 2^lbads */
 678         nd->lbaf[0] = nvstore->sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT;
 679 }
 680
 681 static void
 682 pci_nvme_init_logpages(struct pci_nvme_softc *sc)
 683 {
 684
 685         memset(&sc->err_log, 0, sizeof(sc->err_log));
 686         memset(&sc->health_log, 0, sizeof(sc->health_log));
 687         memset(&sc->fw_log, 0, sizeof(sc->fw_log));
 688         memset(&sc->ns_log, 0, sizeof(sc->ns_log));
 689
 690         /* Set read/write remainder to round up according to spec */
 691         sc->read_dunits_remainder = 999;
 692         sc->write_dunits_remainder = 999;
 693
 694         /* Set nominal Health values checked by implementations */
 695         sc->health_log.temperature = NVME_TEMPERATURE;
 696         sc->health_log.available_spare = 100;
 697         sc->health_log.available_spare_threshold = 10;
 698 }
 699
 700 static void
 701 pci_nvme_init_features(struct pci_nvme_softc *sc)
 702 {
 703         enum nvme_feature       fid;
 704
 705         for (fid = 0; fid < NVME_FID_MAX; fid++) {
 706                 switch (fid) {
 707                 case NVME_FEAT_ARBITRATION:
 708                 case NVME_FEAT_POWER_MANAGEMENT:
 709                 case NVME_FEAT_INTERRUPT_COALESCING: //XXX
 710                 case NVME_FEAT_WRITE_ATOMICITY:
 711                         /* Mandatory but no special handling required */
 712                 //XXX hang - case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG:
 713                 //XXX hang - case NVME_FEAT_HOST_BEHAVIOR_SUPPORT:
 714                 //                this returns a data buffer
 715                         break;
 716                 case NVME_FEAT_TEMPERATURE_THRESHOLD:
 717                         sc->feat[fid].set = nvme_feature_temperature;
 718                         break;
 719                 case NVME_FEAT_ERROR_RECOVERY:
 720                         sc->feat[fid].namespace_specific = true;
 721                         break;
 722                 case NVME_FEAT_NUMBER_OF_QUEUES:
 723                         sc->feat[fid].set = nvme_feature_num_queues;
 724                         break;
 725                 case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
 726                         sc->feat[fid].set = nvme_feature_iv_config;
 727                         break;
 728                 case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
 729                         sc->feat[fid].set = nvme_feature_async_event;
 730                         /* Enable all AENs by default */
 731                         sc->feat[fid].cdw11 = PCI_NVME_AEN_DEFAULT_MASK;
 732                         break;
 733                 default:
 734                         sc->feat[fid].set = nvme_feature_invalid_cb;
 735                         sc->feat[fid].get = nvme_feature_invalid_cb;
 736                 }
 737         }
 738 }
 739
 740 static void
 741 pci_nvme_aer_reset(struct pci_nvme_softc *sc)
 742 {
 743
 744         STAILQ_INIT(&sc->aer_list);
 745         sc->aer_count = 0;
 746 }
 747
 748 static void
 749 pci_nvme_aer_init(struct pci_nvme_softc *sc)
 750 {
 751
 752         pthread_mutex_init(&sc->aer_mtx, NULL);
 753         pci_nvme_aer_reset(sc);
 754 }
 755
 756 static void
 757 pci_nvme_aer_destroy(struct pci_nvme_softc *sc)
 758 {
 759         struct pci_nvme_aer *aer = NULL;
 760
 761         pthread_mutex_lock(&sc->aer_mtx);
 762         while (!STAILQ_EMPTY(&sc->aer_list)) {
 763                 aer = STAILQ_FIRST(&sc->aer_list);
 764                 STAILQ_REMOVE_HEAD(&sc->aer_list, link);
 765                 free(aer);
 766         }
 767         pthread_mutex_unlock(&sc->aer_mtx);
 768
 769         pci_nvme_aer_reset(sc);
 770 }
 771
 772 static bool
 773 pci_nvme_aer_available(struct pci_nvme_softc *sc)
 774 {
 775
 776         return (sc->aer_count != 0);
 777 }
 778
 779 static bool
 780 pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc)
 781 {
 782         struct nvme_controller_data *cd = &sc->ctrldata;
 783
 784         /* AERL is a zero based value while aer_count is one's based */
 785         return (sc->aer_count == (cd->aerl + 1));
 786 }
 787
 788 /*
 789  * Add an Async Event Request
 790  *
 791  * Stores an AER to be returned later if the Controller needs to notify the
 792  * host of an event.
 793  * Note that while the NVMe spec doesn't require Controllers to return AER's
 794  * in order, this implementation does preserve the order.
 795  */
 796 static int
 797 pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid)
 798 {
 799         struct pci_nvme_aer *aer = NULL;
 800
 801         aer = calloc(1, sizeof(struct pci_nvme_aer));
 802         if (aer == NULL)
 803                 return (-1);
 804
 805         /* Save the Command ID for use in the completion message */
 806         aer->cid = cid;
 807
 808         pthread_mutex_lock(&sc->aer_mtx);
 809         sc->aer_count++;
 810         STAILQ_INSERT_TAIL(&sc->aer_list, aer, link);
 811         pthread_mutex_unlock(&sc->aer_mtx);
 812
 813         return (0);
 814 }
 815
 816 /*
 817  * Get an Async Event Request structure
 818  *
 819  * Returns a pointer to an AER previously submitted by the host or NULL if
 820  * no AER's exist. Caller is responsible for freeing the returned struct.
 821  */
 822 static struct pci_nvme_aer *
 823 pci_nvme_aer_get(struct pci_nvme_softc *sc)
 824 {
 825         struct pci_nvme_aer *aer = NULL;
 826
 827         pthread_mutex_lock(&sc->aer_mtx);
 828         aer = STAILQ_FIRST(&sc->aer_list);
 829         if (aer != NULL) {
 830                 STAILQ_REMOVE_HEAD(&sc->aer_list, link);
 831                 sc->aer_count--;
 832         }
 833         pthread_mutex_unlock(&sc->aer_mtx);
 834
 835         return (aer);
 836 }
 837
 838 static void
 839 pci_nvme_aen_reset(struct pci_nvme_softc *sc)
 840 {
 841         uint32_t        atype;
 842
 843         memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen));
 844
 845         for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
 846                 sc->aen[atype].atype = atype;
 847         }
 848 }
 849
 850 static void
 851 pci_nvme_aen_init(struct pci_nvme_softc *sc)
 852 {
 853         char nstr[80];
 854
 855         pci_nvme_aen_reset(sc);
 856
 857         pthread_mutex_init(&sc->aen_mtx, NULL);
 858         pthread_create(&sc->aen_tid, NULL, aen_thr, sc);
 859         snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot,
 860             sc->nsc_pi->pi_func);
 861         pthread_set_name_np(sc->aen_tid, nstr);
 862 }
 863
 864 static void
 865 pci_nvme_aen_destroy(struct pci_nvme_softc *sc)
 866 {
 867
 868         pci_nvme_aen_reset(sc);
 869 }
 870
 871 /* Notify the AEN thread of pending work */
 872 static void
 873 pci_nvme_aen_notify(struct pci_nvme_softc *sc)
 874 {
 875
 876         pthread_cond_signal(&sc->aen_cond);
 877 }
 878
 879 /*
 880  * Post an Asynchronous Event Notification
 881  */
 882 static int32_t
 883 pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype,
 884                 uint32_t event_data)
 885 {
 886         struct pci_nvme_aen *aen;
 887
 888         if (atype >= PCI_NVME_AE_TYPE_MAX) {
 889                 return(EINVAL);
 890         }
 891
 892         pthread_mutex_lock(&sc->aen_mtx);
 893         aen = &sc->aen[atype];
 894
 895         /* Has the controller already posted an event of this type? */
 896         if (aen->posted) {
 897                 pthread_mutex_unlock(&sc->aen_mtx);
 898                 return(EALREADY);
 899         }
 900
 901         aen->event_data = event_data;
 902         aen->posted = true;
 903         pthread_mutex_unlock(&sc->aen_mtx);
 904
 905         pci_nvme_aen_notify(sc);
 906
 907         return(0);
 908 }
 909
 910 static void
 911 pci_nvme_aen_process(struct pci_nvme_softc *sc)
 912 {
 913         struct pci_nvme_aer *aer;
 914         struct pci_nvme_aen *aen;
 915         pci_nvme_async_type atype;
 916         uint32_t mask;
 917         uint16_t status;
 918         uint8_t lid;
 919
 920         assert(pthread_mutex_isowned_np(&sc->aen_mtx));
 921         for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
 922                 aen = &sc->aen[atype];
 923                 /* Previous iterations may have depleted the available AER's */
 924                 if (!pci_nvme_aer_available(sc)) {
 925                         DPRINTF("%s: no AER", __func__);
 926                         break;
 927                 }
 928
 929                 if (!aen->posted) {
 930                         DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype);
 931                         continue;
 932                 }
 933
 934                 status = NVME_SC_SUCCESS;
 935
 936                 /* Is the event masked? */
 937                 mask =
 938                     sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11;
 939
 940                 DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data);
 941                 switch (atype) {
 942                 case PCI_NVME_AE_TYPE_ERROR:
 943                         lid = NVME_LOG_ERROR;
 944                         break;
 945                 case PCI_NVME_AE_TYPE_SMART:
 946                         mask &= 0xff;
 947                         if ((mask & aen->event_data) == 0)
 948                                 continue;
 949                         lid = NVME_LOG_HEALTH_INFORMATION;
 950                         break;
 951                 case PCI_NVME_AE_TYPE_NOTICE:
 952                         if (aen->event_data >= PCI_NVME_AEI_NOTICE_MAX) {
 953                                 EPRINTLN("%s unknown AEN notice type %u",
 954                                     __func__, aen->event_data);
 955                                 status = NVME_SC_INTERNAL_DEVICE_ERROR;
 956                                 break;
 957                         }
 958                         if ((PCI_NVME_AEI_NOTICE_MASK(aen->event_data) & mask) == 0)
 959                                 continue;
 960                         switch (aen->event_data) {
 961                         case PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED:
 962                                 lid = NVME_LOG_CHANGED_NAMESPACE;
 963                                 break;
 964                         case PCI_NVME_AEI_NOTICE_FW_ACTIVATION:
 965                                 lid = NVME_LOG_FIRMWARE_SLOT;
 966                                 break;
 967                         case PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE:
 968                                 lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED;
 969                                 break;
 970                         case PCI_NVME_AEI_NOTICE_ANA_CHANGE:
 971                                 lid = NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS;
 972                                 break;
 973                         case PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE:
 974                                 lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE;
 975                                 break;
 976                         case PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT:
 977                                 lid = NVME_LOG_LBA_STATUS_INFORMATION;
 978                                 break;
 979                         case PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE:
 980                                 lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE;
 981                                 break;
 982                         default:
 983                                 lid = 0;
 984                         }
 985                         break;
 986                 default:
 987                         /* bad type?!? */
 988                         EPRINTLN("%s unknown AEN type %u", __func__, atype);
 989                         status = NVME_SC_INTERNAL_DEVICE_ERROR;
 990                         break;
 991                 }
 992
 993                 aer = pci_nvme_aer_get(sc);
 994                 assert(aer != NULL);
 995
 996                 DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype);
 997                 pci_nvme_cq_update(sc, &sc->compl_queues[0],
 998                     (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */
 999                     aer->cid,
1000                     0,          /* SQID */
1001                     status);
1002
1003                 aen->event_data = 0;
1004                 aen->posted = false;
1005
1006                 pci_generate_msix(sc->nsc_pi, 0);
1007         }
1008 }
1009
1010 static void *
1011 aen_thr(void *arg)
1012 {
1013         struct pci_nvme_softc *sc;
1014
1015         sc = arg;
1016
1017         pthread_mutex_lock(&sc->aen_mtx);
1018         for (;;) {
1019                 pci_nvme_aen_process(sc);
1020                 pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx);
1021         }
1022         pthread_mutex_unlock(&sc->aen_mtx);
1023
1024         pthread_exit(NULL);
1025         return (NULL);
1026 }
1027
1028 static void
1029 pci_nvme_reset_locked(struct pci_nvme_softc *sc)
1030 {
1031         uint32_t i;
1032
1033         DPRINTF("%s", __func__);
1034
1035         sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) |
1036             (1 << NVME_CAP_LO_REG_CQR_SHIFT) |
1037             (60 << NVME_CAP_LO_REG_TO_SHIFT);
1038
1039         sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;
1040
1041         sc->regs.vs = NVME_REV(1,4);    /* NVMe v1.4 */
1042
1043         sc->regs.cc = 0;
1044
1045         assert(sc->submit_queues != NULL);
1046
1047         for (i = 0; i < sc->num_squeues + 1; i++) {
1048                 sc->submit_queues[i].qbase = NULL;
1049                 sc->submit_queues[i].size = 0;
1050                 sc->submit_queues[i].cqid = 0;
1051                 sc->submit_queues[i].tail = 0;
1052                 sc->submit_queues[i].head = 0;
1053         }
1054
1055         assert(sc->compl_queues != NULL);
1056
1057         for (i = 0; i < sc->num_cqueues + 1; i++) {
1058                 sc->compl_queues[i].qbase = NULL;
1059                 sc->compl_queues[i].size = 0;
1060                 sc->compl_queues[i].tail = 0;
1061                 sc->compl_queues[i].head = 0;
1062         }
1063
1064         sc->num_q_is_set = false;
1065
1066         pci_nvme_aer_destroy(sc);
1067         pci_nvme_aen_destroy(sc);
1068
1069         /*
1070          * Clear CSTS.RDY last to prevent the host from enabling Controller
1071          * before cleanup completes
1072          */
1073         sc->regs.csts = 0;
1074 }
1075
1076 static void
1077 pci_nvme_reset(struct pci_nvme_softc *sc)
1078 {
1079         pthread_mutex_lock(&sc->mtx);
1080         pci_nvme_reset_locked(sc);
1081         pthread_mutex_unlock(&sc->mtx);
1082 }
1083
1084 static void
1085 pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc)
1086 {
1087         uint16_t acqs, asqs;
1088
1089         DPRINTF("%s", __func__);
1090
1091         asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1;
1092         sc->submit_queues[0].size = asqs;
1093         sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq,
1094                     sizeof(struct nvme_command) * asqs);
1095
1096         DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p",
1097                 __func__, sc->regs.asq, sc->submit_queues[0].qbase);
1098
1099         acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
1100             NVME_AQA_REG_ACQS_MASK) + 1;
1101         sc->compl_queues[0].size = acqs;
1102         sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,
1103                  sizeof(struct nvme_completion) * acqs);
1104         sc->compl_queues[0].intr_en = NVME_CQ_INTEN;
1105
1106         DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p",
1107                 __func__, sc->regs.acq, sc->compl_queues[0].qbase);
1108 }
1109
1110 static int
1111 nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *b,
1112         size_t len, enum nvme_copy_dir dir)
1113 {
1114         uint8_t *p;
1115         size_t bytes;
1116
1117         if (len > (8 * 1024)) {
1118                 return (-1);
1119         }
1120
1121         /* Copy from the start of prp1 to the end of the physical page */
1122         bytes = PAGE_SIZE - (prp1 & PAGE_MASK);
1123         bytes = MIN(bytes, len);
1124
1125         p = vm_map_gpa(ctx, prp1, bytes);
1126         if (p == NULL) {
1127                 return (-1);
1128         }
1129
1130         if (dir == NVME_COPY_TO_PRP)
1131                 memcpy(p, b, bytes);
1132         else
1133                 memcpy(b, p, bytes);
1134
1135         b += bytes;
1136
1137         len -= bytes;
1138         if (len == 0) {
1139                 return (0);
1140         }
1141
1142         len = MIN(len, PAGE_SIZE);
1143
1144         p = vm_map_gpa(ctx, prp2, len);
1145         if (p == NULL) {
1146                 return (-1);
1147         }
1148
1149         if (dir == NVME_COPY_TO_PRP)
1150                 memcpy(p, b, len);
1151         else
1152                 memcpy(b, p, len);
1153
1154         return (0);
1155 }
1156
1157 /*
1158  * Write a Completion Queue Entry update
1159  *
1160  * Write the completion and update the doorbell value
1161  */
1162 static void
1163 pci_nvme_cq_update(struct pci_nvme_softc *sc,
1164                 struct nvme_completion_queue *cq,
1165                 uint32_t cdw0,
1166                 uint16_t cid,
1167                 uint16_t sqid,
1168                 uint16_t status)
1169 {
1170         struct nvme_submission_queue *sq = &sc->submit_queues[sqid];
1171         struct nvme_completion *cqe;
1172
1173         assert(cq->qbase != NULL);
1174
1175         pthread_mutex_lock(&cq->mtx);
1176
1177         cqe = &cq->qbase[cq->tail];
1178
1179         /* Flip the phase bit */
1180         status |= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK;
1181
1182         cqe->cdw0 = cdw0;
1183         cqe->sqhd = sq->head;
1184         cqe->sqid = sqid;
1185         cqe->cid = cid;
1186         cqe->status = status;
1187
1188         cq->tail++;
1189         if (cq->tail >= cq->size) {
1190                 cq->tail = 0;
1191         }
1192
1193         pthread_mutex_unlock(&cq->mtx);
1194 }
1195
1196 static int
1197 nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
1198         struct nvme_completion* compl)
1199 {
1200         uint16_t qid = command->cdw10 & 0xffff;
1201
1202         DPRINTF("%s DELETE_IO_SQ %u", __func__, qid);
1203         if (qid == 0 || qid > sc->num_squeues ||
1204             (sc->submit_queues[qid].qbase == NULL)) {
1205                 WPRINTF("%s NOT PERMITTED queue id %u / num_squeues %u",
1206                         __func__, qid, sc->num_squeues);
1207                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1208                     NVME_SC_INVALID_QUEUE_IDENTIFIER);
1209                 return (1);
1210         }
1211
1212         sc->submit_queues[qid].qbase = NULL;
1213         sc->submit_queues[qid].cqid = 0;
1214         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1215         return (1);
1216 }
1217
1218 static int
1219 nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
1220         struct nvme_completion* compl)
1221 {
1222         if (command->cdw11 & NVME_CMD_CDW11_PC) {
1223                 uint16_t qid = command->cdw10 & 0xffff;
1224                 struct nvme_submission_queue *nsq;
1225
1226                 if ((qid == 0) || (qid > sc->num_squeues) ||
1227                     (sc->submit_queues[qid].qbase != NULL)) {
1228                         WPRINTF("%s queue index %u > num_squeues %u",
1229                                 __func__, qid, sc->num_squeues);
1230                         pci_nvme_status_tc(&compl->status,
1231                             NVME_SCT_COMMAND_SPECIFIC,
1232                             NVME_SC_INVALID_QUEUE_IDENTIFIER);
1233                         return (1);
1234                 }
1235
1236                 nsq = &sc->submit_queues[qid];
1237                 nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
1238                 DPRINTF("%s size=%u (max=%u)", __func__, nsq->size, sc->max_qentries);
1239                 if ((nsq->size < 2) || (nsq->size > sc->max_qentries)) {
1240                         /*
1241                          * Queues must specify at least two entries
1242                          * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to
1243                          * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec
1244                          */
1245                         pci_nvme_status_tc(&compl->status,
1246                             NVME_SCT_COMMAND_SPECIFIC,
1247                             NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED);
1248                         return (1);
1249                 }
1250                 nsq->head = nsq->tail = 0;
1251
1252                 nsq->cqid = (command->cdw11 >> 16) & 0xffff;
1253                 if ((nsq->cqid == 0) || (nsq->cqid > sc->num_cqueues)) {
1254                         pci_nvme_status_tc(&compl->status,
1255                             NVME_SCT_COMMAND_SPECIFIC,
1256                             NVME_SC_INVALID_QUEUE_IDENTIFIER);
1257                         return (1);
1258                 }
1259
1260                 if (sc->compl_queues[nsq->cqid].qbase == NULL) {
1261                         pci_nvme_status_tc(&compl->status,
1262                             NVME_SCT_COMMAND_SPECIFIC,
1263                             NVME_SC_COMPLETION_QUEUE_INVALID);
1264                         return (1);
1265                 }
1266
1267                 nsq->qpriority = (command->cdw11 >> 1) & 0x03;
1268
1269                 nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
1270                               sizeof(struct nvme_command) * (size_t)nsq->size);
1271
1272                 DPRINTF("%s sq %u size %u gaddr %p cqid %u", __func__,
1273                         qid, nsq->size, nsq->qbase, nsq->cqid);
1274
1275                 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1276
1277                 DPRINTF("%s completed creating IOSQ qid %u",
1278                          __func__, qid);
1279         } else {
1280                 /*
1281                  * Guest sent non-cont submission queue request.
1282                  * This setting is unsupported by this emulation.
1283                  */
1284                 WPRINTF("%s unsupported non-contig (list-based) "
1285                          "create i/o submission queue", __func__);
1286
1287                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1288         }
1289         return (1);
1290 }
1291
1292 static int
1293 nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
1294         struct nvme_completion* compl)
1295 {
1296         uint16_t qid = command->cdw10 & 0xffff;
1297         uint16_t sqid;
1298
1299         DPRINTF("%s DELETE_IO_CQ %u", __func__, qid);
1300         if (qid == 0 || qid > sc->num_cqueues ||
1301             (sc->compl_queues[qid].qbase == NULL)) {
1302                 WPRINTF("%s queue index %u / num_cqueues %u",
1303                         __func__, qid, sc->num_cqueues);
1304                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1305                     NVME_SC_INVALID_QUEUE_IDENTIFIER);
1306                 return (1);
1307         }
1308
1309         /* Deleting an Active CQ is an error */
1310         for (sqid = 1; sqid < sc->num_squeues + 1; sqid++)
1311                 if (sc->submit_queues[sqid].cqid == qid) {
1312                         pci_nvme_status_tc(&compl->status,
1313                             NVME_SCT_COMMAND_SPECIFIC,
1314                             NVME_SC_INVALID_QUEUE_DELETION);
1315                         return (1);
1316                 }
1317
1318         sc->compl_queues[qid].qbase = NULL;
1319         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1320         return (1);
1321 }
1322
1323 static int
1324 nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
1325         struct nvme_completion* compl)
1326 {
1327         struct nvme_completion_queue *ncq;
1328         uint16_t qid = command->cdw10 & 0xffff;
1329
1330         /* Only support Physically Contiguous queues */
1331         if ((command->cdw11 & NVME_CMD_CDW11_PC) == 0) {
1332                 WPRINTF("%s unsupported non-contig (list-based) "
1333                          "create i/o completion queue",
1334                          __func__);
1335
1336                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1337                 return (1);
1338         }
1339
1340         if ((qid == 0) || (qid > sc->num_cqueues) ||
1341             (sc->compl_queues[qid].qbase != NULL)) {
1342                 WPRINTF("%s queue index %u > num_cqueues %u",
1343                         __func__, qid, sc->num_cqueues);
1344                 pci_nvme_status_tc(&compl->status,
1345                     NVME_SCT_COMMAND_SPECIFIC,
1346                     NVME_SC_INVALID_QUEUE_IDENTIFIER);
1347                 return (1);
1348         }
1349
1350         ncq = &sc->compl_queues[qid];
1351         ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1;
1352         ncq->intr_vec = (command->cdw11 >> 16) & 0xffff;
1353         if (ncq->intr_vec > (sc->max_queues + 1)) {
1354                 pci_nvme_status_tc(&compl->status,
1355                     NVME_SCT_COMMAND_SPECIFIC,
1356                     NVME_SC_INVALID_INTERRUPT_VECTOR);
1357                 return (1);
1358         }
1359
1360         ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
1361         if ((ncq->size < 2) || (ncq->size > sc->max_qentries))  {
1362                 /*
1363                  * Queues must specify at least two entries
1364                  * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to
1365                  * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec
1366                  */
1367                 pci_nvme_status_tc(&compl->status,
1368                     NVME_SCT_COMMAND_SPECIFIC,
1369                     NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED);
1370                 return (1);
1371         }
1372         ncq->head = ncq->tail = 0;
1373         ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
1374                      command->prp1,
1375                      sizeof(struct nvme_command) * (size_t)ncq->size);
1376
1377         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1378
1379
1380         return (1);
1381 }
1382
1383 static int
1384 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command,
1385         struct nvme_completion* compl)
1386 {
1387         uint64_t logoff;
1388         uint32_t logsize;
1389         uint8_t logpage = command->cdw10 & 0xFF;
1390
1391         DPRINTF("%s log page %u len %u", __func__, logpage, logsize);
1392
1393         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1394
1395         /*
1396          * Command specifies the number of dwords to return in fields NUMDU
1397          * and NUMDL. This is a zero-based value.
1398          */
1399         logsize = ((command->cdw11 << 16) | (command->cdw10 >> 16)) + 1;
1400         logsize *= sizeof(uint32_t);
1401         logoff  = ((uint64_t)(command->cdw13) << 32) | command->cdw12;
1402
1403         switch (logpage) {
1404         case NVME_LOG_ERROR:
1405                 if (logoff >= sizeof(sc->err_log)) {
1406                         pci_nvme_status_genc(&compl->status,
1407                             NVME_SC_INVALID_FIELD);
1408                         break;
1409                 }
1410
1411                 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1412                     command->prp2, (uint8_t *)&sc->err_log + logoff,
1413                     MIN(logsize - logoff, sizeof(sc->err_log)),
1414                     NVME_COPY_TO_PRP);
1415                 break;
1416         case NVME_LOG_HEALTH_INFORMATION:
1417                 if (logoff >= sizeof(sc->health_log)) {
1418                         pci_nvme_status_genc(&compl->status,
1419                             NVME_SC_INVALID_FIELD);
1420                         break;
1421                 }
1422
1423                 pthread_mutex_lock(&sc->mtx);
1424                 memcpy(&sc->health_log.data_units_read, &sc->read_data_units,
1425                     sizeof(sc->health_log.data_units_read));
1426                 memcpy(&sc->health_log.data_units_written, &sc->write_data_units,
1427                     sizeof(sc->health_log.data_units_written));
1428                 memcpy(&sc->health_log.host_read_commands, &sc->read_commands,
1429                     sizeof(sc->health_log.host_read_commands));
1430                 memcpy(&sc->health_log.host_write_commands, &sc->write_commands,
1431                     sizeof(sc->health_log.host_write_commands));
1432                 pthread_mutex_unlock(&sc->mtx);
1433
1434                 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1435                     command->prp2, (uint8_t *)&sc->health_log + logoff,
1436                     MIN(logsize - logoff, sizeof(sc->health_log)),
1437                     NVME_COPY_TO_PRP);
1438                 break;
1439         case NVME_LOG_FIRMWARE_SLOT:
1440                 if (logoff >= sizeof(sc->fw_log)) {
1441                         pci_nvme_status_genc(&compl->status,
1442                             NVME_SC_INVALID_FIELD);
1443                         break;
1444                 }
1445
1446                 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1447                     command->prp2, (uint8_t *)&sc->fw_log + logoff,
1448                     MIN(logsize - logoff, sizeof(sc->fw_log)),
1449                     NVME_COPY_TO_PRP);
1450                 break;
1451         case NVME_LOG_CHANGED_NAMESPACE:
1452                 if (logoff >= sizeof(sc->ns_log)) {
1453                         pci_nvme_status_genc(&compl->status,
1454                             NVME_SC_INVALID_FIELD);
1455                         break;
1456                 }
1457
1458                 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1459                     command->prp2, (uint8_t *)&sc->ns_log + logoff,
1460                     MIN(logsize - logoff, sizeof(sc->ns_log)),
1461                     NVME_COPY_TO_PRP);
1462                 memset(&sc->ns_log, 0, sizeof(sc->ns_log));
1463                 break;
1464         default:
1465                 DPRINTF("%s get log page %x command not supported",
1466                         __func__, logpage);
1467
1468                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1469                     NVME_SC_INVALID_LOG_PAGE);
1470         }
1471
1472         return (1);
1473 }
1474
1475 static int
1476 nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command,
1477         struct nvme_completion* compl)
1478 {
1479         void *dest;
1480         uint16_t status;
1481
1482         DPRINTF("%s identify 0x%x nsid 0x%x", __func__,
1483                 command->cdw10 & 0xFF, command->nsid);
1484
1485         pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1486
1487         switch (command->cdw10 & 0xFF) {
1488         case 0x00: /* return Identify Namespace data structure */
1489                 /* Global NS only valid with NS Management */
1490                 if (command->nsid == NVME_GLOBAL_NAMESPACE_TAG) {
1491                         pci_nvme_status_genc(&status,
1492                             NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
1493                         break;
1494                 }
1495                 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1496                     command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata),
1497                     NVME_COPY_TO_PRP);
1498                 break;
1499         case 0x01: /* return Identify Controller data structure */
1500                 nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1501                     command->prp2, (uint8_t *)&sc->ctrldata,
1502                     sizeof(sc->ctrldata),
1503                     NVME_COPY_TO_PRP);
1504                 break;
1505         case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */
1506                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
1507                                   sizeof(uint32_t) * 1024);
1508                 /* All unused entries shall be zero */
1509                 bzero(dest, sizeof(uint32_t) * 1024);
1510                 ((uint32_t *)dest)[0] = 1;
1511                 break;
1512         case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */
1513                 if (command->nsid != 1) {
1514                         pci_nvme_status_genc(&status,
1515                             NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
1516                         break;
1517                 }
1518                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
1519                                   sizeof(uint32_t) * 1024);
1520                 /* All bytes after the descriptor shall be zero */
1521                 bzero(dest, sizeof(uint32_t) * 1024);
1522
1523                 /* Return NIDT=1 (i.e. EUI64) descriptor */
1524                 ((uint8_t *)dest)[0] = 1;
1525                 ((uint8_t *)dest)[1] = sizeof(uint64_t);
1526                 bcopy(sc->nsdata.eui64, ((uint8_t *)dest) + 4, sizeof(uint64_t));
1527                 break;
1528         default:
1529                 DPRINTF("%s unsupported identify command requested 0x%x",
1530                          __func__, command->cdw10 & 0xFF);
1531                 pci_nvme_status_genc(&status, NVME_SC_INVALID_FIELD);
1532                 break;
1533         }
1534
1535         compl->status = status;
1536         return (1);
1537 }
1538
1539 static const char *
1540 nvme_fid_to_name(uint8_t fid)
1541 {
1542         const char *name;
1543
1544         switch (fid) {
1545         case NVME_FEAT_ARBITRATION:
1546                 name = "Arbitration";
1547                 break;
1548         case NVME_FEAT_POWER_MANAGEMENT:
1549                 name = "Power Management";
1550                 break;
1551         case NVME_FEAT_LBA_RANGE_TYPE:
1552                 name = "LBA Range Type";
1553                 break;
1554         case NVME_FEAT_TEMPERATURE_THRESHOLD:
1555                 name = "Temperature Threshold";
1556                 break;
1557         case NVME_FEAT_ERROR_RECOVERY:
1558                 name = "Error Recovery";
1559                 break;
1560         case NVME_FEAT_VOLATILE_WRITE_CACHE:
1561                 name = "Volatile Write Cache";
1562                 break;
1563         case NVME_FEAT_NUMBER_OF_QUEUES:
1564                 name = "Number of Queues";
1565                 break;
1566         case NVME_FEAT_INTERRUPT_COALESCING:
1567                 name = "Interrupt Coalescing";
1568                 break;
1569         case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
1570                 name = "Interrupt Vector Configuration";
1571                 break;
1572         case NVME_FEAT_WRITE_ATOMICITY:
1573                 name = "Write Atomicity Normal";
1574                 break;
1575         case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
1576                 name = "Asynchronous Event Configuration";
1577                 break;
1578         case NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION:
1579                 name = "Autonomous Power State Transition";
1580                 break;
1581         case NVME_FEAT_HOST_MEMORY_BUFFER:
1582                 name = "Host Memory Buffer";
1583                 break;
1584         case NVME_FEAT_TIMESTAMP:
1585                 name = "Timestamp";
1586                 break;
1587         case NVME_FEAT_KEEP_ALIVE_TIMER:
1588                 name = "Keep Alive Timer";
1589                 break;
1590         case NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT:
1591                 name = "Host Controlled Thermal Management";
1592                 break;
1593         case NVME_FEAT_NON_OP_POWER_STATE_CONFIG:
1594                 name = "Non-Operation Power State Config";
1595                 break;
1596         case NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG:
1597                 name = "Read Recovery Level Config";
1598                 break;
1599         case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG:
1600                 name = "Predictable Latency Mode Config";
1601                 break;
1602         case NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW:
1603                 name = "Predictable Latency Mode Window";
1604                 break;
1605         case NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES:
1606                 name = "LBA Status Information Report Interval";
1607                 break;
1608         case NVME_FEAT_HOST_BEHAVIOR_SUPPORT:
1609                 name = "Host Behavior Support";
1610                 break;
1611         case NVME_FEAT_SANITIZE_CONFIG:
1612                 name = "Sanitize Config";
1613                 break;
1614         case NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION:
1615                 name = "Endurance Group Event Configuration";
1616                 break;
1617         case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
1618                 name = "Software Progress Marker";
1619                 break;
1620         case NVME_FEAT_HOST_IDENTIFIER:
1621                 name = "Host Identifier";
1622                 break;
1623         case NVME_FEAT_RESERVATION_NOTIFICATION_MASK:
1624                 name = "Reservation Notification Mask";
1625                 break;
1626         case NVME_FEAT_RESERVATION_PERSISTENCE:
1627                 name = "Reservation Persistence";
1628                 break;
1629         case NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG:
1630                 name = "Namespace Write Protection Config";
1631                 break;
1632         default:
1633                 name = "Unknown";
1634                 break;
1635         }
1636
1637         return (name);
1638 }
1639
1640 static void
1641 nvme_feature_invalid_cb(struct pci_nvme_softc *sc,
1642     struct nvme_feature_obj *feat,
1643     struct nvme_command *command,
1644     struct nvme_completion *compl)
1645 {
1646
1647         pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1648 }
1649
1650 static void
1651 nvme_feature_iv_config(struct pci_nvme_softc *sc,
1652     struct nvme_feature_obj *feat,
1653     struct nvme_command *command,
1654     struct nvme_completion *compl)
1655 {
1656         uint32_t i;
1657         uint32_t cdw11 = command->cdw11;
1658         uint16_t iv;
1659         bool cd;
1660
1661         pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1662
1663         iv = cdw11 & 0xffff;
1664         cd = cdw11 & (1 << 16);
1665
1666         if (iv > (sc->max_queues + 1)) {
1667                 return;
1668         }
1669
1670         /* No Interrupt Coalescing (i.e. not Coalescing Disable) for Admin Q */
1671         if ((iv == 0) && !cd)
1672                 return;
1673
1674         /* Requested Interrupt Vector must be used by a CQ */
1675         for (i = 0; i < sc->num_cqueues + 1; i++) {
1676                 if (sc->compl_queues[i].intr_vec == iv) {
1677                         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1678                 }
1679         }
1680 }
1681
1682 #define NVME_ASYNC_EVENT_ENDURANCE_GROUP                (0x4000)
1683 static void
1684 nvme_feature_async_event(struct pci_nvme_softc *sc,
1685     struct nvme_feature_obj *feat,
1686     struct nvme_command *command,
1687     struct nvme_completion *compl)
1688 {
1689
1690         if (command->cdw11 & NVME_ASYNC_EVENT_ENDURANCE_GROUP)
1691                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1692 }
1693
1694 #define NVME_TEMP_THRESH_OVER   0
1695 #define NVME_TEMP_THRESH_UNDER  1
1696 static void
1697 nvme_feature_temperature(struct pci_nvme_softc *sc,
1698     struct nvme_feature_obj *feat,
1699     struct nvme_command *command,
1700     struct nvme_completion *compl)
1701 {
1702         uint16_t        tmpth;  /* Temperature Threshold */
1703         uint8_t         tmpsel; /* Threshold Temperature Select */
1704         uint8_t         thsel;  /* Threshold Type Select */
1705         bool            set_crit = false;
1706
1707         tmpth  = command->cdw11 & 0xffff;
1708         tmpsel = (command->cdw11 >> 16) & 0xf;
1709         thsel  = (command->cdw11 >> 20) & 0x3;
1710
1711         DPRINTF("%s: tmpth=%#x tmpsel=%#x thsel=%#x", __func__, tmpth, tmpsel, thsel);
1712
1713         /* Check for unsupported values */
1714         if (((tmpsel != 0) && (tmpsel != 0xf)) ||
1715             (thsel > NVME_TEMP_THRESH_UNDER)) {
1716                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1717                 return;
1718         }
1719
1720         if (((thsel == NVME_TEMP_THRESH_OVER)  && (NVME_TEMPERATURE >= tmpth)) ||
1721             ((thsel == NVME_TEMP_THRESH_UNDER) && (NVME_TEMPERATURE <= tmpth)))
1722                 set_crit = true;
1723
1724         pthread_mutex_lock(&sc->mtx);
1725         if (set_crit)
1726                 sc->health_log.critical_warning |=
1727                     NVME_CRIT_WARN_ST_TEMPERATURE;
1728         else
1729                 sc->health_log.critical_warning &=
1730                     ~NVME_CRIT_WARN_ST_TEMPERATURE;
1731         pthread_mutex_unlock(&sc->mtx);
1732
1733         if (set_crit)
1734                 pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_SMART,
1735                     sc->health_log.critical_warning);
1736
1737
1738         DPRINTF("%s: set_crit=%c critical_warning=%#x status=%#x", __func__, set_crit ? 'T':'F', sc->health_log.critical_warning, compl->status);
1739 }
1740
1741 static void
1742 nvme_feature_num_queues(struct pci_nvme_softc *sc,
1743     struct nvme_feature_obj *feat,
1744     struct nvme_command *command,
1745     struct nvme_completion *compl)
1746 {
1747         uint16_t nqr;   /* Number of Queues Requested */
1748
1749         if (sc->num_q_is_set) {
1750                 WPRINTF("%s: Number of Queues already set", __func__);
1751                 pci_nvme_status_genc(&compl->status,
1752                     NVME_SC_COMMAND_SEQUENCE_ERROR);
1753                 return;
1754         }
1755
1756         nqr = command->cdw11 & 0xFFFF;
1757         if (nqr == 0xffff) {
1758                 WPRINTF("%s: Illegal NSQR value %#x", __func__, nqr);
1759                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1760                 return;
1761         }
1762
1763         sc->num_squeues = ONE_BASED(nqr);
1764         if (sc->num_squeues > sc->max_queues) {
1765                 DPRINTF("NSQR=%u is greater than max %u", sc->num_squeues,
1766                                         sc->max_queues);
1767                 sc->num_squeues = sc->max_queues;
1768         }
1769
1770         nqr = (command->cdw11 >> 16) & 0xFFFF;
1771         if (nqr == 0xffff) {
1772                 WPRINTF("%s: Illegal NCQR value %#x", __func__, nqr);
1773                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1774                 return;
1775         }
1776
1777         sc->num_cqueues = ONE_BASED(nqr);
1778         if (sc->num_cqueues > sc->max_queues) {
1779                 DPRINTF("NCQR=%u is greater than max %u", sc->num_cqueues,
1780                                         sc->max_queues);
1781                 sc->num_cqueues = sc->max_queues;
1782         }
1783
1784         /* Patch the command value which will be saved on callback's return */
1785         command->cdw11 = NVME_FEATURE_NUM_QUEUES(sc);
1786         compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc);
1787
1788         sc->num_q_is_set = true;
1789 }
1790
1791 static int
1792 nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command,
1793         struct nvme_completion *compl)
1794 {
1795         struct nvme_feature_obj *feat;
1796         uint32_t nsid = command->nsid;
1797         uint8_t fid = command->cdw10 & 0xFF;
1798
1799         DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid));
1800
1801         if (fid >= NVME_FID_MAX) {
1802                 DPRINTF("%s invalid feature 0x%x", __func__, fid);
1803                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1804                 return (1);
1805         }
1806         feat = &sc->feat[fid];
1807
1808         if (feat->namespace_specific && (nsid == NVME_GLOBAL_NAMESPACE_TAG)) {
1809                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1810                 return (1);
1811         }
1812
1813         if (!feat->namespace_specific &&
1814             !((nsid == 0) || (nsid == NVME_GLOBAL_NAMESPACE_TAG))) {
1815                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1816                     NVME_SC_FEATURE_NOT_NS_SPECIFIC);
1817                 return (1);
1818         }
1819
1820         compl->cdw0 = 0;
1821         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1822
1823         if (feat->set)
1824                 feat->set(sc, feat, command, compl);
1825
1826         DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11);
1827         if (compl->status == NVME_SC_SUCCESS) {
1828                 feat->cdw11 = command->cdw11;
1829                 if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) &&
1830                     (command->cdw11 != 0))
1831                         pci_nvme_aen_notify(sc);
1832         }
1833
1834         return (0);
1835 }
1836
1837 #define NVME_FEATURES_SEL_SUPPORTED     0x3
1838 #define NVME_FEATURES_NS_SPECIFIC       (1 << 1)
1839
1840 static int
1841 nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command,
1842         struct nvme_completion* compl)
1843 {
1844         struct nvme_feature_obj *feat;
1845         uint8_t fid = command->cdw10 & 0xFF;
1846         uint8_t sel = (command->cdw10 >> 8) & 0x7;
1847
1848         DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid));
1849
1850         if (fid >= NVME_FID_MAX) {
1851                 DPRINTF("%s invalid feature 0x%x", __func__, fid);
1852                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1853                 return (1);
1854         }
1855
1856         compl->cdw0 = 0;
1857         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1858
1859         feat = &sc->feat[fid];
1860         if (feat->get) {
1861                 feat->get(sc, feat, command, compl);
1862         }
1863
1864         if (compl->status == NVME_SC_SUCCESS) {
1865                 if ((sel == NVME_FEATURES_SEL_SUPPORTED) && feat->namespace_specific)
1866                         compl->cdw0 = NVME_FEATURES_NS_SPECIFIC;
1867                 else
1868                         compl->cdw0 = feat->cdw11;
1869         }
1870
1871         return (0);
1872 }
1873
1874 static int
1875 nvme_opc_format_nvm(struct pci_nvme_softc* sc, struct nvme_command* command,
1876         struct nvme_completion* compl)
1877 {
1878         uint8_t ses, lbaf, pi;
1879
1880         /* Only supports Secure Erase Setting - User Data Erase */
1881         ses = (command->cdw10 >> 9) & 0x7;
1882         if (ses > 0x1) {
1883                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1884                 return (1);
1885         }
1886
1887         /* Only supports a single LBA Format */
1888         lbaf = command->cdw10 & 0xf;
1889         if (lbaf != 0) {
1890                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1891                     NVME_SC_INVALID_FORMAT);
1892                 return (1);
1893         }
1894
1895         /* Doesn't support Protection Infomation */
1896         pi = (command->cdw10 >> 5) & 0x7;
1897         if (pi != 0) {
1898                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1899                 return (1);
1900         }
1901
1902         if (sc->nvstore.type == NVME_STOR_RAM) {
1903                 if (sc->nvstore.ctx)
1904                         free(sc->nvstore.ctx);
1905                 sc->nvstore.ctx = calloc(1, sc->nvstore.size);
1906                 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1907         } else {
1908                 struct pci_nvme_ioreq *req;
1909                 int err;
1910
1911                 req = pci_nvme_get_ioreq(sc);
1912                 if (req == NULL) {
1913                         pci_nvme_status_genc(&compl->status,
1914                             NVME_SC_INTERNAL_DEVICE_ERROR);
1915                         WPRINTF("%s: unable to allocate IO req", __func__);
1916                         return (1);
1917                 }
1918                 req->nvme_sq = &sc->submit_queues[0];
1919                 req->sqid = 0;
1920                 req->opc = command->opc;
1921                 req->cid = command->cid;
1922                 req->nsid = command->nsid;
1923
1924                 req->io_req.br_offset = 0;
1925                 req->io_req.br_resid = sc->nvstore.size;
1926                 req->io_req.br_callback = pci_nvme_io_done;
1927
1928                 err = blockif_delete(sc->nvstore.ctx, &req->io_req);
1929                 if (err) {
1930                         pci_nvme_status_genc(&compl->status,
1931                             NVME_SC_INTERNAL_DEVICE_ERROR);
1932                         pci_nvme_release_ioreq(sc, req);
1933                 } else
1934                         compl->status = NVME_NO_STATUS;
1935         }
1936
1937         return (1);
1938 }
1939
1940 static int
1941 nvme_opc_abort(struct pci_nvme_softc* sc, struct nvme_command* command,
1942         struct nvme_completion* compl)
1943 {
1944         DPRINTF("%s submission queue %u, command ID 0x%x", __func__,
1945                 command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF);
1946
1947         /* TODO: search for the command ID and abort it */
1948
1949         compl->cdw0 = 1;
1950         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
1951         return (1);
1952 }
1953
1954 static int
1955 nvme_opc_async_event_req(struct pci_nvme_softc* sc,
1956         struct nvme_command* command, struct nvme_completion* compl)
1957 {
1958         DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__,
1959             sc->aer_count, sc->ctrldata.aerl, command->cid);
1960
1961         /* Don't exceed the Async Event Request Limit (AERL). */
1962         if (pci_nvme_aer_limit_reached(sc)) {
1963                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
1964                                 NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
1965                 return (1);
1966         }
1967
1968         if (pci_nvme_aer_add(sc, command->cid)) {
1969                 pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC,
1970                                 NVME_SC_INTERNAL_DEVICE_ERROR);
1971                 return (1);
1972         }
1973
1974         /*
1975          * Raise events when they happen based on the Set Features cmd.
1976          * These events happen async, so only set completion successful if
1977          * there is an event reflective of the request to get event.
1978          */
1979         compl->status = NVME_NO_STATUS;
1980         pci_nvme_aen_notify(sc);
1981
1982         return (0);
1983 }
1984
1985 static void
1986 pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
1987 {
1988         struct nvme_completion compl;
1989         struct nvme_command *cmd;
1990         struct nvme_submission_queue *sq;
1991         struct nvme_completion_queue *cq;
1992         uint16_t sqhead;
1993
1994         DPRINTF("%s index %u", __func__, (uint32_t)value);
1995
1996         sq = &sc->submit_queues[0];
1997         cq = &sc->compl_queues[0];
1998
1999         pthread_mutex_lock(&sq->mtx);
2000
2001         sqhead = sq->head;
2002         DPRINTF("sqhead %u, tail %u", sqhead, sq->tail);
2003
2004         while (sqhead != atomic_load_acq_short(&sq->tail)) {
2005                 cmd = &(sq->qbase)[sqhead];
2006                 compl.cdw0 = 0;
2007                 compl.status = 0;
2008
2009                 switch (cmd->opc) {
2010                 case NVME_OPC_DELETE_IO_SQ:
2011                         DPRINTF("%s command DELETE_IO_SQ", __func__);
2012                         nvme_opc_delete_io_sq(sc, cmd, &compl);
2013                         break;
2014                 case NVME_OPC_CREATE_IO_SQ:
2015                         DPRINTF("%s command CREATE_IO_SQ", __func__);
2016                         nvme_opc_create_io_sq(sc, cmd, &compl);
2017                         break;
2018                 case NVME_OPC_DELETE_IO_CQ:
2019                         DPRINTF("%s command DELETE_IO_CQ", __func__);
2020                         nvme_opc_delete_io_cq(sc, cmd, &compl);
2021                         break;
2022                 case NVME_OPC_CREATE_IO_CQ:
2023                         DPRINTF("%s command CREATE_IO_CQ", __func__);
2024                         nvme_opc_create_io_cq(sc, cmd, &compl);
2025                         break;
2026                 case NVME_OPC_GET_LOG_PAGE:
2027                         DPRINTF("%s command GET_LOG_PAGE", __func__);
2028                         nvme_opc_get_log_page(sc, cmd, &compl);
2029                         break;
2030                 case NVME_OPC_IDENTIFY:
2031                         DPRINTF("%s command IDENTIFY", __func__);
2032                         nvme_opc_identify(sc, cmd, &compl);
2033                         break;
2034                 case NVME_OPC_ABORT:
2035                         DPRINTF("%s command ABORT", __func__);
2036                         nvme_opc_abort(sc, cmd, &compl);
2037                         break;
2038                 case NVME_OPC_SET_FEATURES:
2039                         DPRINTF("%s command SET_FEATURES", __func__);
2040                         nvme_opc_set_features(sc, cmd, &compl);
2041                         break;
2042                 case NVME_OPC_GET_FEATURES:
2043                         DPRINTF("%s command GET_FEATURES", __func__);
2044                         nvme_opc_get_features(sc, cmd, &compl);
2045                         break;
2046                 case NVME_OPC_FIRMWARE_ACTIVATE:
2047                         DPRINTF("%s command FIRMWARE_ACTIVATE", __func__);
2048                         pci_nvme_status_tc(&compl.status,
2049                             NVME_SCT_COMMAND_SPECIFIC,
2050                             NVME_SC_INVALID_FIRMWARE_SLOT);
2051                         break;
2052                 case NVME_OPC_ASYNC_EVENT_REQUEST:
2053                         DPRINTF("%s command ASYNC_EVENT_REQ", __func__);
2054                         nvme_opc_async_event_req(sc, cmd, &compl);
2055                         break;
2056                 case NVME_OPC_FORMAT_NVM:
2057                         DPRINTF("%s command FORMAT_NVM", __func__);
2058                         if ((sc->ctrldata.oacs &
2059                             (1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) {
2060                                 pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
2061                                 break;
2062                         }
2063                         nvme_opc_format_nvm(sc, cmd, &compl);
2064                         break;
2065                 case NVME_OPC_SECURITY_SEND:
2066                 case NVME_OPC_SECURITY_RECEIVE:
2067                 case NVME_OPC_SANITIZE:
2068                 case NVME_OPC_GET_LBA_STATUS:
2069                         DPRINTF("%s command OPC=%#x (unsupported)", __func__,
2070                             cmd->opc);
2071                         /* Valid but unsupported opcodes */
2072                         pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_FIELD);
2073                         break;
2074                 default:
2075                         DPRINTF("%s command OPC=%#X (not implemented)",
2076                             __func__,
2077                             cmd->opc);
2078                         pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
2079                 }
2080                 sqhead = (sqhead + 1) % sq->size;
2081
2082                 if (NVME_COMPLETION_VALID(compl)) {
2083                         pci_nvme_cq_update(sc, &sc->compl_queues[0],
2084                             compl.cdw0,
2085                             cmd->cid,
2086                             0,          /* SQID */
2087                             compl.status);
2088                 }
2089         }
2090
2091         DPRINTF("setting sqhead %u", sqhead);
2092         sq->head = sqhead;
2093
2094         if (cq->head != cq->tail)
2095                 pci_generate_msix(sc->nsc_pi, 0);
2096
2097         pthread_mutex_unlock(&sq->mtx);
2098 }
2099
2100 /*
2101  * Update the Write and Read statistics reported in SMART data
2102  *
2103  * NVMe defines "data unit" as thousand's of 512 byte blocks and is rounded up.
2104  * E.g. 1 data unit is 1 - 1,000 512 byte blocks. 3 data units are 2,001 - 3,000
2105  * 512 byte blocks. Rounding up is acheived by initializing the remainder to 999.
2106  */
2107 static void
2108 pci_nvme_stats_write_read_update(struct pci_nvme_softc *sc, uint8_t opc,
2109     size_t bytes, uint16_t status)
2110 {
2111
2112         pthread_mutex_lock(&sc->mtx);
2113         switch (opc) {
2114         case NVME_OPC_WRITE:
2115                 sc->write_commands++;
2116                 if (status != NVME_SC_SUCCESS)
2117                         break;
2118                 sc->write_dunits_remainder += (bytes / 512);
2119                 while (sc->write_dunits_remainder >= 1000) {
2120                         sc->write_data_units++;
2121                         sc->write_dunits_remainder -= 1000;
2122                 }
2123                 break;
2124         case NVME_OPC_READ:
2125                 sc->read_commands++;
2126                 if (status != NVME_SC_SUCCESS)
2127                         break;
2128                 sc->read_dunits_remainder += (bytes / 512);
2129                 while (sc->read_dunits_remainder >= 1000) {
2130                         sc->read_data_units++;
2131                         sc->read_dunits_remainder -= 1000;
2132                 }
2133                 break;
2134         default:
2135                 DPRINTF("%s: Invalid OPC 0x%02x for stats", __func__, opc);
2136                 break;
2137         }
2138         pthread_mutex_unlock(&sc->mtx);
2139 }
2140
2141 /*
2142  * Check if the combination of Starting LBA (slba) and number of blocks
2143  * exceeds the range of the underlying storage.
2144  *
2145  * Because NVMe specifies the SLBA in blocks as a uint64_t and blockif stores
2146  * the capacity in bytes as a uint64_t, care must be taken to avoid integer
2147  * overflow.
2148  */
2149 static bool
2150 pci_nvme_out_of_range(struct pci_nvme_blockstore *nvstore, uint64_t slba,
2151     uint32_t nblocks)
2152 {
2153         size_t  offset, bytes;
2154
2155         /* Overflow check of multiplying Starting LBA by the sector size */
2156         if (slba >> (64 - nvstore->sectsz_bits))
2157                 return (true);
2158
2159         offset = slba << nvstore->sectsz_bits;
2160         bytes = nblocks << nvstore->sectsz_bits;
2161
2162         /* Overflow check of Number of Logical Blocks */
2163         if ((nvstore->size <= offset) || ((nvstore->size - offset) < bytes))
2164                 return (true);
2165
2166         return (false);
2167 }
2168
2169 static int
2170 pci_nvme_append_iov_req(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req,
2171         uint64_t gpaddr, size_t size, int do_write, uint64_t lba)
2172 {
2173         int iovidx;
2174
2175         if (req == NULL)
2176                 return (-1);
2177
2178         if (req->io_req.br_iovcnt == NVME_MAX_IOVEC) {
2179                 return (-1);
2180         }
2181
2182         /* concatenate contig block-iovs to minimize number of iovs */
2183         if ((req->prev_gpaddr + req->prev_size) == gpaddr) {
2184                 iovidx = req->io_req.br_iovcnt - 1;
2185
2186                 req->io_req.br_iov[iovidx].iov_base =
2187                     paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
2188                                      req->prev_gpaddr, size);
2189
2190                 req->prev_size += size;
2191                 req->io_req.br_resid += size;
2192
2193                 req->io_req.br_iov[iovidx].iov_len = req->prev_size;
2194         } else {
2195                 iovidx = req->io_req.br_iovcnt;
2196                 if (iovidx == 0) {
2197                         req->io_req.br_offset = lba;
2198                         req->io_req.br_resid = 0;
2199                         req->io_req.br_param = req;
2200                 }
2201
2202                 req->io_req.br_iov[iovidx].iov_base =
2203                     paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
2204                                      gpaddr, size);
2205
2206                 req->io_req.br_iov[iovidx].iov_len = size;
2207
2208                 req->prev_gpaddr = gpaddr;
2209                 req->prev_size = size;
2210                 req->io_req.br_resid += size;
2211
2212                 req->io_req.br_iovcnt++;
2213         }
2214
2215         return (0);
2216 }
2217
2218 static void
2219 pci_nvme_set_completion(struct pci_nvme_softc *sc,
2220         struct nvme_submission_queue *sq, int sqid, uint16_t cid,
2221         uint32_t cdw0, uint16_t status)
2222 {
2223         struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
2224
2225         DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x",
2226                  __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
2227                  NVME_STATUS_GET_SC(status));
2228
2229         pci_nvme_cq_update(sc, cq,
2230             0,          /* CDW0 */
2231             cid,
2232             sqid,
2233             status);
2234
2235         if (cq->head != cq->tail) {
2236                 if (cq->intr_en & NVME_CQ_INTEN) {
2237                         pci_generate_msix(sc->nsc_pi, cq->intr_vec);
2238                 } else {
2239                         DPRINTF("%s: CQ%u interrupt disabled",
2240                                                 __func__, sq->cqid);
2241                 }
2242         }
2243 }
2244
2245 static void
2246 pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req)
2247 {
2248         req->sc = NULL;
2249         req->nvme_sq = NULL;
2250         req->sqid = 0;
2251
2252         pthread_mutex_lock(&sc->mtx);
2253
2254         STAILQ_INSERT_TAIL(&sc->ioreqs_free, req, link);
2255         sc->pending_ios--;
2256
2257         /* when no more IO pending, can set to ready if device reset/enabled */
2258         if (sc->pending_ios == 0 &&
2259             NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts)))
2260                 sc->regs.csts |= NVME_CSTS_RDY;
2261
2262         pthread_mutex_unlock(&sc->mtx);
2263
2264         sem_post(&sc->iosemlock);
2265 }
2266
2267 static struct pci_nvme_ioreq *
2268 pci_nvme_get_ioreq(struct pci_nvme_softc *sc)
2269 {
2270         struct pci_nvme_ioreq *req = NULL;
2271
2272         sem_wait(&sc->iosemlock);
2273         pthread_mutex_lock(&sc->mtx);
2274
2275         req = STAILQ_FIRST(&sc->ioreqs_free);
2276         assert(req != NULL);
2277         STAILQ_REMOVE_HEAD(&sc->ioreqs_free, link);
2278
2279         req->sc = sc;
2280
2281         sc->pending_ios++;
2282
2283         pthread_mutex_unlock(&sc->mtx);
2284
2285         req->io_req.br_iovcnt = 0;
2286         req->io_req.br_offset = 0;
2287         req->io_req.br_resid = 0;
2288         req->io_req.br_param = req;
2289         req->prev_gpaddr = 0;
2290         req->prev_size = 0;
2291
2292         return req;
2293 }
2294
2295 static void
2296 pci_nvme_io_done(struct blockif_req *br, int err)
2297 {
2298         struct pci_nvme_ioreq *req = br->br_param;
2299         struct nvme_submission_queue *sq = req->nvme_sq;
2300         uint16_t code, status;
2301
2302         DPRINTF("%s error %d %s", __func__, err, strerror(err));
2303
2304         /* TODO return correct error */
2305         code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;
2306         pci_nvme_status_genc(&status, code);
2307
2308         pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status);
2309         pci_nvme_stats_write_read_update(req->sc, req->opc,
2310             req->bytes, status);
2311         pci_nvme_release_ioreq(req->sc, req);
2312 }
2313
2314 /*
2315  * Implements the Flush command. The specification states:
2316  *    If a volatile write cache is not present, Flush commands complete
2317  *    successfully and have no effect
2318  * in the description of the Volatile Write Cache (VWC) field of the Identify
2319  * Controller data. Therefore, set status to Success if the command is
2320  * not supported (i.e. RAM or as indicated by the blockif).
2321  */
2322 static bool
2323 nvme_opc_flush(struct pci_nvme_softc *sc,
2324     struct nvme_command *cmd,
2325     struct pci_nvme_blockstore *nvstore,
2326     struct pci_nvme_ioreq *req,
2327     uint16_t *status)
2328 {
2329         bool pending = false;
2330
2331         if (nvstore->type == NVME_STOR_RAM) {
2332                 pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2333         } else {
2334                 int err;
2335
2336                 req->io_req.br_callback = pci_nvme_io_done;
2337
2338                 err = blockif_flush(nvstore->ctx, &req->io_req);
2339                 switch (err) {
2340                 case 0:
2341                         pending = true;
2342                         break;
2343                 case EOPNOTSUPP:
2344                         pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2345                         break;
2346                 default:
2347                         pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2348                 }
2349         }
2350
2351         return (pending);
2352 }
2353
2354 static uint16_t
2355 nvme_write_read_ram(struct pci_nvme_softc *sc,
2356     struct pci_nvme_blockstore *nvstore,
2357     uint64_t prp1, uint64_t prp2,
2358     size_t offset, uint64_t bytes,
2359     bool is_write)
2360 {
2361         uint8_t *buf = nvstore->ctx;
2362         enum nvme_copy_dir dir;
2363         uint16_t status;
2364
2365         if (is_write)
2366                 dir = NVME_COPY_TO_PRP;
2367         else
2368                 dir = NVME_COPY_FROM_PRP;
2369
2370         if (nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, prp1, prp2,
2371             buf + offset, bytes, dir))
2372                 pci_nvme_status_genc(&status,
2373                     NVME_SC_DATA_TRANSFER_ERROR);
2374         else
2375                 pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
2376
2377         return (status);
2378 }
2379
2380 static uint16_t
2381 nvme_write_read_blockif(struct pci_nvme_softc *sc,
2382     struct pci_nvme_blockstore *nvstore,
2383     struct pci_nvme_ioreq *req,
2384     uint64_t prp1, uint64_t prp2,
2385     size_t offset, uint64_t bytes,
2386     bool is_write)
2387 {
2388         uint64_t size;
2389         int err;
2390         uint16_t status = NVME_NO_STATUS;
2391
2392         size = MIN(PAGE_SIZE - (prp1 % PAGE_SIZE), bytes);
2393         if (pci_nvme_append_iov_req(sc, req, prp1,
2394             size, is_write, offset)) {
2395                 pci_nvme_status_genc(&status,
2396                     NVME_SC_DATA_TRANSFER_ERROR);
2397                 goto out;
2398         }
2399
2400         offset += size;
2401         bytes  -= size;
2402
2403         if (bytes == 0) {
2404                 ;
2405         } else if (bytes <= PAGE_SIZE) {
2406                 size = bytes;
2407                 if (pci_nvme_append_iov_req(sc, req, prp2,
2408                     size, is_write, offset)) {
2409                         pci_nvme_status_genc(&status,
2410                             NVME_SC_DATA_TRANSFER_ERROR);
2411                         goto out;
2412                 }
2413         } else {
2414                 void *vmctx = sc->nsc_pi->pi_vmctx;
2415                 uint64_t *prp_list = &prp2;
2416                 uint64_t *last = prp_list;
2417
2418                 /* PRP2 is pointer to a physical region page list */
2419                 while (bytes) {
2420                         /* Last entry in list points to the next list */
2421                         if ((prp_list == last) && (bytes > PAGE_SIZE)) {
2422                                 uint64_t prp = *prp_list;
2423
2424                                 prp_list = paddr_guest2host(vmctx, prp,
2425                                     PAGE_SIZE - (prp % PAGE_SIZE));
2426                                 last = prp_list + (NVME_PRP2_ITEMS - 1);
2427                         }
2428
2429                         size = MIN(bytes, PAGE_SIZE);
2430
2431                         if (pci_nvme_append_iov_req(sc, req, *prp_list,
2432                             size, is_write, offset)) {
2433                                 pci_nvme_status_genc(&status,
2434                                     NVME_SC_DATA_TRANSFER_ERROR);
2435                                 goto out;
2436                         }
2437
2438                         offset += size;
2439                         bytes  -= size;
2440
2441                         prp_list++;
2442                 }
2443         }
2444         req->io_req.br_callback = pci_nvme_io_done;
2445         if (is_write)
2446                 err = blockif_write(nvstore->ctx, &req->io_req);
2447         else
2448                 err = blockif_read(nvstore->ctx, &req->io_req);
2449
2450         if (err)
2451                 pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR);
2452 out:
2453         return (status);
2454 }
2455
2456 static bool
2457 nvme_opc_write_read(struct pci_nvme_softc *sc,
2458     struct nvme_command *cmd,
2459     struct pci_nvme_blockstore *nvstore,
2460     struct pci_nvme_ioreq *req,
2461     uint16_t *status)
2462 {
2463         uint64_t lba, nblocks, bytes;
2464         size_t offset;
2465         bool is_write = cmd->opc == NVME_OPC_WRITE;
2466         bool pending = false;
2467
2468         lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
2469         nblocks = (cmd->cdw12 & 0xFFFF) + 1;
2470
2471         if (pci_nvme_out_of_range(nvstore, lba, nblocks)) {
2472                 WPRINTF("%s command would exceed LBA range(slba=%#lx nblocks=%#lx)",
2473                     __func__, lba, nblocks);
2474                 pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE);
2475                 goto out;
2476         }
2477
2478         bytes  = nblocks << nvstore->sectsz_bits;
2479         if (bytes > NVME_MAX_DATA_SIZE) {
2480                 WPRINTF("%s command would exceed MDTS", __func__);
2481                 pci_nvme_status_genc(status, NVME_SC_INVALID_FIELD);
2482                 goto out;
2483         }
2484
2485         offset = lba << nvstore->sectsz_bits;
2486
2487         req->bytes = bytes;
2488         req->io_req.br_offset = lba;
2489
2490         /* PRP bits 1:0 must be zero */
2491         cmd->prp1 &= ~0x3UL;
2492         cmd->prp2 &= ~0x3UL;
2493
2494         if (nvstore->type == NVME_STOR_RAM) {
2495                 *status = nvme_write_read_ram(sc, nvstore, cmd->prp1,
2496                     cmd->prp2, offset, bytes, is_write);
2497         } else {
2498                 *status = nvme_write_read_blockif(sc, nvstore, req,
2499                     cmd->prp1, cmd->prp2, offset, bytes, is_write);
2500
2501                 if (*status == NVME_NO_STATUS)
2502                         pending = true;
2503         }
2504 out:
2505         if (!pending)
2506                 pci_nvme_stats_write_read_update(sc, cmd->opc, bytes, *status);
2507
2508         return (pending);
2509 }
2510
2511 static void
2512 pci_nvme_dealloc_sm(struct blockif_req *br, int err)
2513 {
2514         struct pci_nvme_ioreq *req = br->br_param;
2515         struct pci_nvme_softc *sc = req->sc;
2516         bool done = true;
2517         uint16_t status;
2518
2519         if (err) {
2520                 pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR);
2521         } else if ((req->prev_gpaddr + 1) == (req->prev_size)) {
2522                 pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
2523         } else {
2524                 struct iovec *iov = req->io_req.br_iov;
2525
2526                 req->prev_gpaddr++;
2527                 iov += req->prev_gpaddr;
2528
2529                 /* The iov_* values already include the sector size */
2530                 req->io_req.br_offset = (off_t)iov->iov_base;
2531                 req->io_req.br_resid = iov->iov_len;
2532                 if (blockif_delete(sc->nvstore.ctx, &req->io_req)) {
2533                         pci_nvme_status_genc(&status,
2534                             NVME_SC_INTERNAL_DEVICE_ERROR);
2535                 } else
2536                         done = false;
2537         }
2538
2539         if (done) {
2540                 pci_nvme_set_completion(sc, req->nvme_sq, req->sqid,
2541                     req->cid, 0, status);
2542                 pci_nvme_release_ioreq(sc, req);
2543         }
2544 }
2545
2546 static bool
2547 nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc,
2548     struct nvme_command *cmd,
2549     struct pci_nvme_blockstore *nvstore,
2550     struct pci_nvme_ioreq *req,
2551     uint16_t *status)
2552 {
2553         struct nvme_dsm_range *range;
2554         uint32_t nr, r, non_zero, dr;
2555         int err;
2556         bool pending = false;
2557
2558         if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) {
2559                 pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE);
2560                 goto out;
2561         }
2562
2563         nr = cmd->cdw10 & 0xff;
2564
2565         /* copy locally because a range entry could straddle PRPs */
2566         range = calloc(1, NVME_MAX_DSM_TRIM);
2567         if (range == NULL) {
2568                 pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2569                 goto out;
2570         }
2571         nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, cmd->prp1, cmd->prp2,
2572             (uint8_t *)range, NVME_MAX_DSM_TRIM, NVME_COPY_FROM_PRP);
2573
2574         /* Check for invalid ranges and the number of non-zero lengths */
2575         non_zero = 0;
2576         for (r = 0; r <= nr; r++) {
2577                 if (pci_nvme_out_of_range(nvstore,
2578                     range[r].starting_lba, range[r].length)) {
2579                         pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE);
2580                         goto out;
2581                 }
2582                 if (range[r].length != 0)
2583                         non_zero++;
2584         }
2585
2586         if (cmd->cdw11 & NVME_DSM_ATTR_DEALLOCATE) {
2587                 size_t offset, bytes;
2588                 int sectsz_bits = sc->nvstore.sectsz_bits;
2589
2590                 /*
2591                  * DSM calls are advisory only, and compliant controllers
2592                  * may choose to take no actions (i.e. return Success).
2593                  */
2594                 if (!nvstore->deallocate) {
2595                         pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2596                         goto out;
2597                 }
2598
2599                 /* If all ranges have a zero length, return Success */
2600                 if (non_zero == 0) {
2601                         pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2602                         goto out;
2603                 }
2604
2605                 if (req == NULL) {
2606                         pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2607                         goto out;
2608                 }
2609
2610                 offset = range[0].starting_lba << sectsz_bits;
2611                 bytes = range[0].length << sectsz_bits;
2612
2613                 /*
2614                  * If the request is for more than a single range, store
2615                  * the ranges in the br_iov. Optimize for the common case
2616                  * of a single range.
2617                  *
2618                  * Note that NVMe Number of Ranges is a zero based value
2619                  */
2620                 req->io_req.br_iovcnt = 0;
2621                 req->io_req.br_offset = offset;
2622                 req->io_req.br_resid = bytes;
2623
2624                 if (nr == 0) {
2625                         req->io_req.br_callback = pci_nvme_io_done;
2626                 } else {
2627                         struct iovec *iov = req->io_req.br_iov;
2628
2629                         for (r = 0, dr = 0; r <= nr; r++) {
2630                                 offset = range[r].starting_lba << sectsz_bits;
2631                                 bytes = range[r].length << sectsz_bits;
2632                                 if (bytes == 0)
2633                                         continue;
2634
2635                                 if ((nvstore->size - offset) < bytes) {
2636                                         pci_nvme_status_genc(status,
2637                                             NVME_SC_LBA_OUT_OF_RANGE);
2638                                         goto out;
2639                                 }
2640                                 iov[dr].iov_base = (void *)offset;
2641                                 iov[dr].iov_len = bytes;
2642                                 dr++;
2643                         }
2644                         req->io_req.br_callback = pci_nvme_dealloc_sm;
2645
2646                         /*
2647                          * Use prev_gpaddr to track the current entry and
2648                          * prev_size to track the number of entries
2649                          */
2650                         req->prev_gpaddr = 0;
2651                         req->prev_size = dr;
2652                 }
2653
2654                 err = blockif_delete(nvstore->ctx, &req->io_req);
2655                 if (err)
2656                         pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2657                 else
2658                         pending = true;
2659         }
2660 out:
2661         free(range);
2662         return (pending);
2663 }
2664
2665 static void
2666 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx)
2667 {
2668         struct nvme_submission_queue *sq;
2669         uint16_t status;
2670         uint16_t sqhead;
2671
2672         /* handle all submissions up to sq->tail index */
2673         sq = &sc->submit_queues[idx];
2674
2675         pthread_mutex_lock(&sq->mtx);
2676
2677         sqhead = sq->head;
2678         DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p",
2679                  idx, sqhead, sq->tail, sq->qbase);
2680
2681         while (sqhead != atomic_load_acq_short(&sq->tail)) {
2682                 struct nvme_command *cmd;
2683                 struct pci_nvme_ioreq *req;
2684                 uint32_t nsid;
2685                 bool pending;
2686
2687                 pending = false;
2688                 req = NULL;
2689                 status = 0;
2690
2691                 cmd = &sq->qbase[sqhead];
2692                 sqhead = (sqhead + 1) % sq->size;
2693
2694                 nsid = le32toh(cmd->nsid);
2695                 if ((nsid == 0) || (nsid > sc->ctrldata.nn)) {
2696                         pci_nvme_status_genc(&status,
2697                             NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
2698                         status |=
2699                             NVME_STATUS_DNR_MASK << NVME_STATUS_DNR_SHIFT;
2700                         goto complete;
2701                 }
2702
2703                 req = pci_nvme_get_ioreq(sc);
2704                 if (req == NULL) {
2705                         pci_nvme_status_genc(&status,
2706                             NVME_SC_INTERNAL_DEVICE_ERROR);
2707                         WPRINTF("%s: unable to allocate IO req", __func__);
2708                         goto complete;
2709                 }
2710                 req->nvme_sq = sq;
2711                 req->sqid = idx;
2712                 req->opc = cmd->opc;
2713                 req->cid = cmd->cid;
2714                 req->nsid = cmd->nsid;
2715
2716                 switch (cmd->opc) {
2717                 case NVME_OPC_FLUSH:
2718                         pending = nvme_opc_flush(sc, cmd, &sc->nvstore,
2719                             req, &status);
2720                         break;
2721                 case NVME_OPC_WRITE:
2722                 case NVME_OPC_READ:
2723                         pending = nvme_opc_write_read(sc, cmd, &sc->nvstore,
2724                             req, &status);
2725                         break;
2726                 case NVME_OPC_WRITE_ZEROES:
2727                         /* TODO: write zeroes
2728                         WPRINTF("%s write zeroes lba 0x%lx blocks %u",
2729                                 __func__, lba, cmd->cdw12 & 0xFFFF); */
2730                         pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
2731                         break;
2732                 case NVME_OPC_DATASET_MANAGEMENT:
2733                         pending = nvme_opc_dataset_mgmt(sc, cmd, &sc->nvstore,
2734                             req, &status);
2735                         break;
2736                 default:
2737                         WPRINTF("%s unhandled io command 0x%x",
2738                             __func__, cmd->opc);
2739                         pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE);
2740                 }
2741 complete:
2742                 if (!pending) {
2743                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
2744                             status);
2745                         if (req != NULL)
2746                                 pci_nvme_release_ioreq(sc, req);
2747                 }
2748         }
2749
2750         sq->head = sqhead;
2751
2752         pthread_mutex_unlock(&sq->mtx);
2753 }
2754
2755 static void
2756 pci_nvme_handle_doorbell(struct vmctx *ctx, struct pci_nvme_softc* sc,
2757         uint64_t idx, int is_sq, uint64_t value)
2758 {
2759         DPRINTF("nvme doorbell %lu, %s, val 0x%lx",
2760                 idx, is_sq ? "SQ" : "CQ", value & 0xFFFF);
2761
2762         if (is_sq) {
2763                 if (idx > sc->num_squeues) {
2764                         WPRINTF("%s queue index %lu overflow from "
2765                                  "guest (max %u)",
2766                                  __func__, idx, sc->num_squeues);
2767                         return;
2768                 }
2769
2770                 atomic_store_short(&sc->submit_queues[idx].tail,
2771                                    (uint16_t)value);
2772
2773                 if (idx == 0) {
2774                         pci_nvme_handle_admin_cmd(sc, value);
2775                 } else {
2776                         /* submission queue; handle new entries in SQ */
2777                         if (idx > sc->num_squeues) {
2778                                 WPRINTF("%s SQ index %lu overflow from "
2779                                          "guest (max %u)",
2780                                          __func__, idx, sc->num_squeues);
2781                                 return;
2782                         }
2783                         pci_nvme_handle_io_cmd(sc, (uint16_t)idx);
2784                 }
2785         } else {
2786                 if (idx > sc->num_cqueues) {
2787                         WPRINTF("%s queue index %lu overflow from "
2788                                  "guest (max %u)",
2789                                  __func__, idx, sc->num_cqueues);
2790                         return;
2791                 }
2792
2793                 atomic_store_short(&sc->compl_queues[idx].head,
2794                                 (uint16_t)value);
2795         }
2796 }
2797
2798 static void
2799 pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)
2800 {
2801         const char *s = iswrite ? "WRITE" : "READ";
2802
2803         switch (offset) {
2804         case NVME_CR_CAP_LOW:
2805                 DPRINTF("%s %s NVME_CR_CAP_LOW", func, s);
2806                 break;
2807         case NVME_CR_CAP_HI:
2808                 DPRINTF("%s %s NVME_CR_CAP_HI", func, s);
2809                 break;
2810         case NVME_CR_VS:
2811                 DPRINTF("%s %s NVME_CR_VS", func, s);
2812                 break;
2813         case NVME_CR_INTMS:
2814                 DPRINTF("%s %s NVME_CR_INTMS", func, s);
2815                 break;
2816         case NVME_CR_INTMC:
2817                 DPRINTF("%s %s NVME_CR_INTMC", func, s);
2818                 break;
2819         case NVME_CR_CC:
2820                 DPRINTF("%s %s NVME_CR_CC", func, s);
2821                 break;
2822         case NVME_CR_CSTS:
2823                 DPRINTF("%s %s NVME_CR_CSTS", func, s);
2824                 break;
2825         case NVME_CR_NSSR:
2826                 DPRINTF("%s %s NVME_CR_NSSR", func, s);
2827                 break;
2828         case NVME_CR_AQA:
2829                 DPRINTF("%s %s NVME_CR_AQA", func, s);
2830                 break;
2831         case NVME_CR_ASQ_LOW:
2832                 DPRINTF("%s %s NVME_CR_ASQ_LOW", func, s);
2833                 break;
2834         case NVME_CR_ASQ_HI:
2835                 DPRINTF("%s %s NVME_CR_ASQ_HI", func, s);
2836                 break;
2837         case NVME_CR_ACQ_LOW:
2838                 DPRINTF("%s %s NVME_CR_ACQ_LOW", func, s);
2839                 break;
2840         case NVME_CR_ACQ_HI:
2841                 DPRINTF("%s %s NVME_CR_ACQ_HI", func, s);
2842                 break;
2843         default:
2844                 DPRINTF("unknown nvme bar-0 offset 0x%lx", offset);
2845         }
2846
2847 }
2848
2849 static void
2850 pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc,
2851         uint64_t offset, int size, uint64_t value)
2852 {
2853         uint32_t ccreg;
2854
2855         if (offset >= NVME_DOORBELL_OFFSET) {
2856                 uint64_t belloffset = offset - NVME_DOORBELL_OFFSET;
2857                 uint64_t idx = belloffset / 8; /* door bell size = 2*int */
2858                 int is_sq = (belloffset % 8) < 4;
2859
2860                 if (belloffset > ((sc->max_queues+1) * 8 - 4)) {
2861                         WPRINTF("guest attempted an overflow write offset "
2862                                  "0x%lx, val 0x%lx in %s",
2863                                  offset, value, __func__);
2864                         return;
2865                 }
2866
2867                 pci_nvme_handle_doorbell(ctx, sc, idx, is_sq, value);
2868                 return;
2869         }
2870
2871         DPRINTF("nvme-write offset 0x%lx, size %d, value 0x%lx",
2872                 offset, size, value);
2873
2874         if (size != 4) {
2875                 WPRINTF("guest wrote invalid size %d (offset 0x%lx, "
2876                          "val 0x%lx) to bar0 in %s",
2877                          size, offset, value, __func__);
2878                 /* TODO: shutdown device */
2879                 return;
2880         }
2881
2882         pci_nvme_bar0_reg_dumps(__func__, offset, 1);
2883
2884         pthread_mutex_lock(&sc->mtx);
2885
2886         switch (offset) {
2887         case NVME_CR_CAP_LOW:
2888         case NVME_CR_CAP_HI:
2889                 /* readonly */
2890                 break;
2891         case NVME_CR_VS:
2892                 /* readonly */
2893                 break;
2894         case NVME_CR_INTMS:
2895                 /* MSI-X, so ignore */
2896                 break;
2897         case NVME_CR_INTMC:
2898                 /* MSI-X, so ignore */
2899                 break;
2900         case NVME_CR_CC:
2901                 ccreg = (uint32_t)value;
2902
2903                 DPRINTF("%s NVME_CR_CC en %x css %x shn %x iosqes %u "
2904                          "iocqes %u",
2905                         __func__,
2906                          NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg),
2907                          NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg),
2908                          NVME_CC_GET_IOCQES(ccreg));
2909
2910                 if (NVME_CC_GET_SHN(ccreg)) {
2911                         /* perform shutdown - flush out data to backend */
2912                         sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK <<
2913                             NVME_CSTS_REG_SHST_SHIFT);
2914                         sc->regs.csts |= NVME_SHST_COMPLETE <<
2915                             NVME_CSTS_REG_SHST_SHIFT;
2916                 }
2917                 if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) {
2918                         if (NVME_CC_GET_EN(ccreg) == 0)
2919                                 /* transition 1-> causes controller reset */
2920                                 pci_nvme_reset_locked(sc);
2921                         else
2922                                 pci_nvme_init_controller(ctx, sc);
2923                 }
2924
2925                 /* Insert the iocqes, iosqes and en bits from the write */
2926                 sc->regs.cc &= ~NVME_CC_WRITE_MASK;
2927                 sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK;
2928                 if (NVME_CC_GET_EN(ccreg) == 0) {
2929                         /* Insert the ams, mps and css bit fields */
2930                         sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK;
2931                         sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK;
2932                         sc->regs.csts &= ~NVME_CSTS_RDY;
2933                 } else if (sc->pending_ios == 0) {
2934                         sc->regs.csts |= NVME_CSTS_RDY;
2935                 }
2936                 break;
2937         case NVME_CR_CSTS:
2938                 break;
2939         case NVME_CR_NSSR:
2940                 /* ignore writes; don't support subsystem reset */
2941                 break;
2942         case NVME_CR_AQA:
2943                 sc->regs.aqa = (uint32_t)value;
2944                 break;
2945         case NVME_CR_ASQ_LOW:
2946                 sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) |
2947                                (0xFFFFF000 & value);
2948                 break;
2949         case NVME_CR_ASQ_HI:
2950                 sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) |
2951                                (value << 32);
2952                 break;
2953         case NVME_CR_ACQ_LOW:
2954                 sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) |
2955                                (0xFFFFF000 & value);
2956                 break;
2957         case NVME_CR_ACQ_HI:
2958                 sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) |
2959                                (value << 32);
2960                 break;
2961         default:
2962                 DPRINTF("%s unknown offset 0x%lx, value 0x%lx size %d",
2963                          __func__, offset, value, size);
2964         }
2965         pthread_mutex_unlock(&sc->mtx);
2966 }
2967
2968 static void
2969 pci_nvme_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
2970                 int baridx, uint64_t offset, int size, uint64_t value)
2971 {
2972         struct pci_nvme_softc* sc = pi->pi_arg;
2973
2974         if (baridx == pci_msix_table_bar(pi) ||
2975             baridx == pci_msix_pba_bar(pi)) {
2976                 DPRINTF("nvme-write baridx %d, msix: off 0x%lx, size %d, "
2977                          " value 0x%lx", baridx, offset, size, value);
2978
2979                 pci_emul_msix_twrite(pi, offset, size, value);
2980                 return;
2981         }
2982
2983         switch (baridx) {
2984         case 0:
2985                 pci_nvme_write_bar_0(ctx, sc, offset, size, value);
2986                 break;
2987
2988         default:
2989                 DPRINTF("%s unknown baridx %d, val 0x%lx",
2990                          __func__, baridx, value);
2991         }
2992 }
2993
2994 static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc,
2995         uint64_t offset, int size)
2996 {
2997         uint64_t value;
2998
2999         pci_nvme_bar0_reg_dumps(__func__, offset, 0);
3000
3001         if (offset < NVME_DOORBELL_OFFSET) {
3002                 void *p = &(sc->regs);
3003                 pthread_mutex_lock(&sc->mtx);
3004                 memcpy(&value, (void *)((uintptr_t)p + offset), size);
3005                 pthread_mutex_unlock(&sc->mtx);
3006         } else {
3007                 value = 0;
3008                 WPRINTF("pci_nvme: read invalid offset %ld", offset);
3009         }
3010
3011         switch (size) {
3012         case 1:
3013                 value &= 0xFF;
3014                 break;
3015         case 2:
3016                 value &= 0xFFFF;
3017                 break;
3018         case 4:
3019                 value &= 0xFFFFFFFF;
3020                 break;
3021         }
3022
3023         DPRINTF("   nvme-read offset 0x%lx, size %d -> value 0x%x",
3024                  offset, size, (uint32_t)value);
3025
3026         return (value);
3027 }
3028
3029
3030
3031 static uint64_t
3032 pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx,
3033     uint64_t offset, int size)
3034 {
3035         struct pci_nvme_softc* sc = pi->pi_arg;
3036
3037         if (baridx == pci_msix_table_bar(pi) ||
3038             baridx == pci_msix_pba_bar(pi)) {
3039                 DPRINTF("nvme-read bar: %d, msix: regoff 0x%lx, size %d",
3040                         baridx, offset, size);
3041
3042                 return pci_emul_msix_tread(pi, offset, size);
3043         }
3044
3045         switch (baridx) {
3046         case 0:
3047                 return pci_nvme_read_bar_0(sc, offset, size);
3048
3049         default:
3050                 DPRINTF("unknown bar %d, 0x%lx", baridx, offset);
3051         }
3052
3053         return (0);
3054 }
3055
3056 static int
3057 pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl)
3058 {
3059         char bident[sizeof("XX:X:X")];
3060         const char *value;
3061         uint32_t sectsz;
3062
3063         sc->max_queues = NVME_QUEUES;
3064         sc->max_qentries = NVME_MAX_QENTRIES;
3065         sc->ioslots = NVME_IOSLOTS;
3066         sc->num_squeues = sc->max_queues;
3067         sc->num_cqueues = sc->max_queues;
3068         sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
3069         sectsz = 0;
3070         snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
3071                  "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
3072
3073         value = get_config_value_node(nvl, "maxq");
3074         if (value != NULL)
3075                 sc->max_queues = atoi(value);
3076         value = get_config_value_node(nvl, "qsz");
3077         if (value != NULL) {
3078                 sc->max_qentries = atoi(value);
3079                 if (sc->max_qentries <= 0) {
3080                         EPRINTLN("nvme: Invalid qsz option %d",
3081                             sc->max_qentries);
3082                         return (-1);
3083                 }
3084         }
3085         value = get_config_value_node(nvl, "ioslots");
3086         if (value != NULL) {
3087                 sc->ioslots = atoi(value);
3088                 if (sc->ioslots <= 0) {
3089                         EPRINTLN("Invalid ioslots option %d", sc->ioslots);
3090                         return (-1);
3091                 }
3092         }
3093         value = get_config_value_node(nvl, "sectsz");
3094         if (value != NULL)
3095                 sectsz = atoi(value);
3096         value = get_config_value_node(nvl, "ser");
3097         if (value != NULL) {
3098                 /*
3099                  * This field indicates the Product Serial Number in
3100                  * 7-bit ASCII, unused bytes should be space characters.
3101                  * Ref: NVMe v1.3c.
3102                  */
3103                 cpywithpad((char *)sc->ctrldata.sn,
3104                     sizeof(sc->ctrldata.sn), value, ' ');
3105         }
3106         value = get_config_value_node(nvl, "eui64");
3107         if (value != NULL)
3108                 sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0));
3109         value = get_config_value_node(nvl, "dsm");
3110         if (value != NULL) {
3111                 if (strcmp(value, "auto") == 0)
3112                         sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
3113                 else if (strcmp(value, "enable") == 0)
3114                         sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE;
3115                 else if (strcmp(value, "disable") == 0)
3116                         sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE;
3117         }
3118
3119         value = get_config_value_node(nvl, "ram");
3120         if (value != NULL) {
3121                 uint64_t sz = strtoull(value, NULL, 10);
3122
3123                 sc->nvstore.type = NVME_STOR_RAM;
3124                 sc->nvstore.size = sz * 1024 * 1024;
3125                 sc->nvstore.ctx = calloc(1, sc->nvstore.size);
3126                 sc->nvstore.sectsz = 4096;
3127                 sc->nvstore.sectsz_bits = 12;
3128                 if (sc->nvstore.ctx == NULL) {
3129                         EPRINTLN("nvme: Unable to allocate RAM");
3130                         return (-1);
3131                 }
3132         } else {
3133                 snprintf(bident, sizeof(bident), "%d:%d",
3134                     sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
3135                 sc->nvstore.ctx = blockif_open(nvl, bident);
3136                 if (sc->nvstore.ctx == NULL) {
3137                         EPRINTLN("nvme: Could not open backing file: %s",
3138                             strerror(errno));
3139                         return (-1);
3140                 }
3141                 sc->nvstore.type = NVME_STOR_BLOCKIF;
3142                 sc->nvstore.size = blockif_size(sc->nvstore.ctx);
3143         }
3144
3145         if (sectsz == 512 || sectsz == 4096 || sectsz == 8192)
3146                 sc->nvstore.sectsz = sectsz;
3147         else if (sc->nvstore.type != NVME_STOR_RAM)
3148                 sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx);
3149         for (sc->nvstore.sectsz_bits = 9;
3150              (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;
3151              sc->nvstore.sectsz_bits++);
3152
3153         if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES)
3154                 sc->max_queues = NVME_QUEUES;
3155
3156         return (0);
3157 }
3158
3159 static void
3160 pci_nvme_resized(struct blockif_ctxt *bctxt, void *arg, size_t new_size)
3161 {
3162         struct pci_nvme_softc *sc;
3163         struct pci_nvme_blockstore *nvstore;
3164         struct nvme_namespace_data *nd;
3165
3166         sc = arg;
3167         nvstore = &sc->nvstore;
3168         nd = &sc->nsdata;
3169
3170         nvstore->size = new_size;
3171         pci_nvme_init_nsdata_size(nvstore, nd);
3172
3173         /* Add changed NSID to list */
3174         sc->ns_log.ns[0] = 1;
3175         sc->ns_log.ns[1] = 0;
3176
3177         pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_NOTICE,
3178             PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED);
3179 }
3180
3181 static int
3182 pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, nvlist_t *nvl)
3183 {
3184         struct pci_nvme_softc *sc;
3185         uint32_t pci_membar_sz;
3186         int     error;
3187
3188         error = 0;
3189
3190         sc = calloc(1, sizeof(struct pci_nvme_softc));
3191         pi->pi_arg = sc;
3192         sc->nsc_pi = pi;
3193
3194         error = pci_nvme_parse_config(sc, nvl);
3195         if (error < 0)
3196                 goto done;
3197         else
3198                 error = 0;
3199
3200         STAILQ_INIT(&sc->ioreqs_free);
3201         sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq));
3202         for (int i = 0; i < sc->ioslots; i++) {
3203                 STAILQ_INSERT_TAIL(&sc->ioreqs_free, &sc->ioreqs[i], link);
3204         }
3205
3206         pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A);
3207         pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D);
3208         pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
3209         pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM);
3210         pci_set_cfgdata8(pi, PCIR_PROGIF,
3211                          PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0);
3212
3213         /*
3214          * Allocate size of NVMe registers + doorbell space for all queues.
3215          *
3216          * The specification requires a minimum memory I/O window size of 16K.
3217          * The Windows driver will refuse to start a device with a smaller
3218          * window.
3219          */
3220         pci_membar_sz = sizeof(struct nvme_registers) +
3221             2 * sizeof(uint32_t) * (sc->max_queues + 1);
3222         pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN);
3223
3224         DPRINTF("nvme membar size: %u", pci_membar_sz);
3225
3226         error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz);
3227         if (error) {
3228                 WPRINTF("%s pci alloc mem bar failed", __func__);
3229                 goto done;
3230         }
3231
3232         error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR);
3233         if (error) {
3234                 WPRINTF("%s pci add msixcap failed", __func__);
3235                 goto done;
3236         }
3237
3238         error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP);
3239         if (error) {
3240                 WPRINTF("%s pci add Express capability failed", __func__);
3241                 goto done;
3242         }
3243
3244         pthread_mutex_init(&sc->mtx, NULL);
3245         sem_init(&sc->iosemlock, 0, sc->ioslots);
3246         blockif_register_resize_callback(sc->nvstore.ctx, pci_nvme_resized, sc);
3247
3248         pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues);
3249         /*
3250          * Controller data depends on Namespace data so initialize Namespace
3251          * data first.
3252          */
3253         pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore);
3254         pci_nvme_init_ctrldata(sc);
3255         pci_nvme_init_logpages(sc);
3256         pci_nvme_init_features(sc);
3257
3258         pci_nvme_aer_init(sc);
3259         pci_nvme_aen_init(sc);
3260
3261         pci_nvme_reset(sc);
3262
3263         pci_lintr_request(pi);
3264
3265 done:
3266         return (error);
3267 }
3268
3269 static int
3270 pci_nvme_legacy_config(nvlist_t *nvl, const char *opts)
3271 {
3272         char *cp, *ram;
3273
3274         if (opts == NULL)
3275                 return (0);
3276
3277         if (strncmp(opts, "ram=", 4) == 0) {
3278                 cp = strchr(opts, ',');
3279                 if (cp == NULL) {
3280                         set_config_value_node(nvl, "ram", opts + 4);
3281                         return (0);
3282                 }
3283                 ram = strndup(opts + 4, cp - opts - 4);
3284                 set_config_value_node(nvl, "ram", ram);
3285                 free(ram);
3286                 return (pci_parse_legacy_config(nvl, cp + 1));
3287         } else
3288                 return (blockif_legacy_config(nvl, opts));
3289 }
3290
3291 static const struct pci_devemu pci_de_nvme = {
3292         .pe_emu =       "nvme",
3293         .pe_init =      pci_nvme_init,
3294         .pe_legacy_config = pci_nvme_legacy_config,
3295         .pe_barwrite =  pci_nvme_write,
3296         .pe_barread =   pci_nvme_read
3297 };
3298 PCI_EMUL_SET(pci_de_nvme);