usr.sbin/bhyve/pci_nvme.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2017 Shunsuke Mie
   5  * Copyright (c) 2018 Leon Dang
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * bhyve PCIe-NVMe device emulation.
  31  *
  32  * options:
  33  *  -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z
  34  *
  35  *  accepted devpath:
  36  *    /dev/blockdev
  37  *    /path/to/image
  38  *    ram=size_in_MiB
  39  *
  40  *  maxq    = max number of queues
  41  *  qsz     = max elements in each queue
  42  *  ioslots = max number of concurrent io requests
  43  *  sectsz  = sector size (defaults to blockif sector size)
  44  *  ser     = serial number (20-chars max)
  45  *
  46  */
  47
  48 /* TODO:
  49     - create async event for smart and log
  50     - intr coalesce
  51  */
  52
  53 #include <sys/cdefs.h>
  54 __FBSDID("$FreeBSD$");
  55
  56 #include <sys/types.h>
  57
  58 #include <assert.h>
  59 #include <pthread.h>
  60 #include <semaphore.h>
  61 #include <stdbool.h>
  62 #include <stddef.h>
  63 #include <stdint.h>
  64 #include <stdio.h>
  65 #include <stdlib.h>
  66 #include <string.h>
  67
  68 #include <machine/atomic.h>
  69 #include <machine/vmm.h>
  70 #include <vmmapi.h>
  71
  72 #include <dev/nvme/nvme.h>
  73
  74 #include "bhyverun.h"
  75 #include "block_if.h"
  76 #include "pci_emul.h"
  77
  78
  79 static int nvme_debug = 0;
  80 #define DPRINTF(params) if (nvme_debug) printf params
  81 #define WPRINTF(params) printf params
  82
  83 /* defaults; can be overridden */
  84 #define NVME_MSIX_BAR           4
  85
  86 #define NVME_IOSLOTS            8
  87
  88 #define NVME_QUEUES             16
  89 #define NVME_MAX_QENTRIES       2048
  90
  91 #define NVME_PRP2_ITEMS         (PAGE_SIZE/sizeof(uint64_t))
  92 #define NVME_MAX_BLOCKIOVS      512
  93
  94 /* helpers */
  95
  96 /* Convert a zero-based value into a one-based value */
  97 #define ONE_BASED(zero)         ((zero) + 1)
  98 /* Convert a one-based value into a zero-based value */
  99 #define ZERO_BASED(one)         ((one)  - 1)
 100
 101 /* Encode number of SQ's and CQ's for Set/Get Features */
 102 #define NVME_FEATURE_NUM_QUEUES(sc) \
 103         (ZERO_BASED((sc)->num_squeues) & 0xffff) | \
 104         (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16;
 105
 106 #define NVME_DOORBELL_OFFSET    offsetof(struct nvme_registers, doorbell)
 107
 108 enum nvme_controller_register_offsets {
 109         NVME_CR_CAP_LOW = 0x00,
 110         NVME_CR_CAP_HI  = 0x04,
 111         NVME_CR_VS      = 0x08,
 112         NVME_CR_INTMS   = 0x0c,
 113         NVME_CR_INTMC   = 0x10,
 114         NVME_CR_CC      = 0x14,
 115         NVME_CR_CSTS    = 0x1c,
 116         NVME_CR_NSSR    = 0x20,
 117         NVME_CR_AQA     = 0x24,
 118         NVME_CR_ASQ_LOW = 0x28,
 119         NVME_CR_ASQ_HI  = 0x2c,
 120         NVME_CR_ACQ_LOW = 0x30,
 121         NVME_CR_ACQ_HI  = 0x34,
 122 };
 123
 124 enum nvme_cmd_cdw11 {
 125         NVME_CMD_CDW11_PC  = 0x0001,
 126         NVME_CMD_CDW11_IEN = 0x0002,
 127         NVME_CMD_CDW11_IV  = 0xFFFF0000,
 128 };
 129
 130 #define NVME_CQ_INTEN   0x01
 131 #define NVME_CQ_INTCOAL 0x02
 132
 133 struct nvme_completion_queue {
 134         struct nvme_completion *qbase;
 135         uint32_t        size;
 136         uint16_t        tail; /* nvme progress */
 137         uint16_t        head; /* guest progress */
 138         uint16_t        intr_vec;
 139         uint32_t        intr_en;
 140         pthread_mutex_t mtx;
 141 };
 142
 143 struct nvme_submission_queue {
 144         struct nvme_command *qbase;
 145         uint32_t        size;
 146         uint16_t        head; /* nvme progress */
 147         uint16_t        tail; /* guest progress */
 148         uint16_t        cqid; /* completion queue id */
 149         int             busy; /* queue is being processed */
 150         int             qpriority;
 151 };
 152
 153 enum nvme_storage_type {
 154         NVME_STOR_BLOCKIF = 0,
 155         NVME_STOR_RAM = 1,
 156 };
 157
 158 struct pci_nvme_blockstore {
 159         enum nvme_storage_type type;
 160         void            *ctx;
 161         uint64_t        size;
 162         uint32_t        sectsz;
 163         uint32_t        sectsz_bits;
 164 };
 165
 166 struct pci_nvme_ioreq {
 167         struct pci_nvme_softc *sc;
 168         struct pci_nvme_ioreq *next;
 169         struct nvme_submission_queue *nvme_sq;
 170         uint16_t        sqid;
 171
 172         /* command information */
 173         uint16_t        opc;
 174         uint16_t        cid;
 175         uint32_t        nsid;
 176
 177         uint64_t        prev_gpaddr;
 178         size_t          prev_size;
 179
 180         /*
 181          * lock if all iovs consumed (big IO);
 182          * complete transaction before continuing
 183          */
 184         pthread_mutex_t mtx;
 185         pthread_cond_t  cv;
 186
 187         struct blockif_req io_req;
 188
 189         /* pad to fit up to 512 page descriptors from guest IO request */
 190         struct iovec    iovpadding[NVME_MAX_BLOCKIOVS-BLOCKIF_IOV_MAX];
 191 };
 192
 193 struct pci_nvme_softc {
 194         struct pci_devinst *nsc_pi;
 195
 196         pthread_mutex_t mtx;
 197
 198         struct nvme_registers regs;
 199
 200         struct nvme_namespace_data  nsdata;
 201         struct nvme_controller_data ctrldata;
 202
 203         struct pci_nvme_blockstore nvstore;
 204
 205         uint16_t        max_qentries;   /* max entries per queue */
 206         uint32_t        max_queues;     /* max number of IO SQ's or CQ's */
 207         uint32_t        num_cqueues;
 208         uint32_t        num_squeues;
 209
 210         struct pci_nvme_ioreq *ioreqs;
 211         struct pci_nvme_ioreq *ioreqs_free; /* free list of ioreqs */
 212         uint32_t        pending_ios;
 213         uint32_t        ioslots;
 214         sem_t           iosemlock;
 215
 216         /*
 217          * Memory mapped Submission and Completion queues
 218          * Each array includes both Admin and IO queues
 219          */
 220         struct nvme_completion_queue *compl_queues;
 221         struct nvme_submission_queue *submit_queues;
 222
 223         /* controller features */
 224         uint32_t        intr_coales_aggr_time;   /* 0x08: uS to delay intr */
 225         uint32_t        intr_coales_aggr_thresh; /* 0x08: compl-Q entries */
 226         uint32_t        async_ev_config;         /* 0x0B: async event config */
 227 };
 228
 229
 230 static void pci_nvme_io_partial(struct blockif_req *br, int err);
 231
 232 /* Controller Configuration utils */
 233 #define NVME_CC_GET_EN(cc) \
 234         ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK)
 235 #define NVME_CC_GET_CSS(cc) \
 236         ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK)
 237 #define NVME_CC_GET_SHN(cc) \
 238         ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK)
 239 #define NVME_CC_GET_IOSQES(cc) \
 240         ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK)
 241 #define NVME_CC_GET_IOCQES(cc) \
 242         ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK)
 243
 244 #define NVME_CC_WRITE_MASK \
 245         ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \
 246          (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \
 247          (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT))
 248
 249 #define NVME_CC_NEN_WRITE_MASK \
 250         ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \
 251          (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \
 252          (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT))
 253
 254 /* Controller Status utils */
 255 #define NVME_CSTS_GET_RDY(sts) \
 256         ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK)
 257
 258 #define NVME_CSTS_RDY   (1 << NVME_CSTS_REG_RDY_SHIFT)
 259
 260 /* Completion Queue status word utils */
 261 #define NVME_STATUS_P   (1 << NVME_STATUS_P_SHIFT)
 262 #define NVME_STATUS_MASK \
 263         ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\
 264          (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT))
 265
 266 static __inline void
 267 cpywithpad(char *dst, size_t dst_size, const char *src, char pad)
 268 {
 269         size_t len;
 270
 271         len = strnlen(src, dst_size);
 272         memset(dst, pad, dst_size);
 273         memcpy(dst, src, len);
 274 }
 275
 276 static __inline void
 277 pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code)
 278 {
 279
 280         *status &= ~NVME_STATUS_MASK;
 281         *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT |
 282                 (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT;
 283 }
 284
 285 static __inline void
 286 pci_nvme_status_genc(uint16_t *status, uint16_t code)
 287 {
 288
 289         pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);
 290 }
 291
 292 static __inline void
 293 pci_nvme_toggle_phase(uint16_t *status, int prev)
 294 {
 295
 296         if (prev)
 297                 *status &= ~NVME_STATUS_P;
 298         else
 299                 *status |= NVME_STATUS_P;
 300 }
 301
 302 static void
 303 pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)
 304 {
 305         struct nvme_controller_data *cd = &sc->ctrldata;
 306
 307         cd->vid = 0xFB5D;
 308         cd->ssvid = 0x0000;
 309
 310         cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' ');
 311         cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' ');
 312
 313         /* Num of submission commands that we can handle at a time (2^rab) */
 314         cd->rab   = 4;
 315
 316         /* FreeBSD OUI */
 317         cd->ieee[0] = 0x58;
 318         cd->ieee[1] = 0x9c;
 319         cd->ieee[2] = 0xfc;
 320
 321         cd->mic = 0;
 322
 323         cd->mdts = 9;   /* max data transfer size (2^mdts * CAP.MPSMIN) */
 324
 325         cd->ver = 0x00010300;
 326
 327         cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT;
 328         cd->acl = 2;
 329         cd->aerl = 4;
 330
 331         cd->lpa = 0;    /* TODO: support some simple things like SMART */
 332         cd->elpe = 0;   /* max error log page entries */
 333         cd->npss = 1;   /* number of power states support */
 334
 335         /* Warning Composite Temperature Threshold */
 336         cd->wctemp = 0x0157;
 337
 338         cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) |
 339             (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT);
 340         cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) |
 341             (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT);
 342         cd->nn = 1;     /* number of namespaces */
 343
 344         cd->fna = 0x03;
 345
 346         cd->power_state[0].mp = 10;
 347 }
 348
 349 static void
 350 pci_nvme_init_nsdata(struct pci_nvme_softc *sc)
 351 {
 352         struct nvme_namespace_data *nd;
 353
 354         nd = &sc->nsdata;
 355
 356         nd->nsze = sc->nvstore.size / sc->nvstore.sectsz;
 357         nd->ncap = nd->nsze;
 358         nd->nuse = nd->nsze;
 359
 360         /* Get LBA and backstore information from backing store */
 361         nd->nlbaf = 1;
 362         /* LBA data-sz = 2^lbads */
 363         nd->lbaf[0] = sc->nvstore.sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT;
 364
 365         nd->flbas = 0;
 366 }
 367
 368 static void
 369 pci_nvme_reset_locked(struct pci_nvme_softc *sc)
 370 {
 371         DPRINTF(("%s\r\n", __func__));
 372
 373         sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) |
 374             (1 << NVME_CAP_LO_REG_CQR_SHIFT) |
 375             (60 << NVME_CAP_LO_REG_TO_SHIFT);
 376
 377         sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;
 378
 379         sc->regs.vs = 0x00010300;       /* NVMe v1.3 */
 380
 381         sc->regs.cc = 0;
 382         sc->regs.csts = 0;
 383
 384         sc->num_cqueues = sc->num_squeues = sc->max_queues;
 385         if (sc->submit_queues != NULL) {
 386                 for (int i = 0; i < sc->num_squeues + 1; i++) {
 387                         /*
 388                          * The Admin Submission Queue is at index 0.
 389                          * It must not be changed at reset otherwise the
 390                          * emulation will be out of sync with the guest.
 391                          */
 392                         if (i != 0) {
 393                                 sc->submit_queues[i].qbase = NULL;
 394                                 sc->submit_queues[i].size = 0;
 395                                 sc->submit_queues[i].cqid = 0;
 396                         }
 397                         sc->submit_queues[i].tail = 0;
 398                         sc->submit_queues[i].head = 0;
 399                         sc->submit_queues[i].busy = 0;
 400                 }
 401         } else
 402                 sc->submit_queues = calloc(sc->num_squeues + 1,
 403                                         sizeof(struct nvme_submission_queue));
 404
 405         if (sc->compl_queues != NULL) {
 406                 for (int i = 0; i < sc->num_cqueues + 1; i++) {
 407                         /* See Admin Submission Queue note above */
 408                         if (i != 0) {
 409                                 sc->compl_queues[i].qbase = NULL;
 410                                 sc->compl_queues[i].size = 0;
 411                         }
 412
 413                         sc->compl_queues[i].tail = 0;
 414                         sc->compl_queues[i].head = 0;
 415                 }
 416         } else {
 417                 sc->compl_queues = calloc(sc->num_cqueues + 1,
 418                                         sizeof(struct nvme_completion_queue));
 419
 420                 for (int i = 0; i < sc->num_cqueues + 1; i++)
 421                         pthread_mutex_init(&sc->compl_queues[i].mtx, NULL);
 422         }
 423 }
 424
 425 static void
 426 pci_nvme_reset(struct pci_nvme_softc *sc)
 427 {
 428         pthread_mutex_lock(&sc->mtx);
 429         pci_nvme_reset_locked(sc);
 430         pthread_mutex_unlock(&sc->mtx);
 431 }
 432
 433 static void
 434 pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc)
 435 {
 436         uint16_t acqs, asqs;
 437
 438         DPRINTF(("%s\r\n", __func__));
 439
 440         asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1;
 441         sc->submit_queues[0].size = asqs;
 442         sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq,
 443                     sizeof(struct nvme_command) * asqs);
 444
 445         DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p\r\n",
 446                 __func__, sc->regs.asq, sc->submit_queues[0].qbase));
 447
 448         acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
 449             NVME_AQA_REG_ACQS_MASK) + 1;
 450         sc->compl_queues[0].size = acqs;
 451         sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,
 452                  sizeof(struct nvme_completion) * acqs);
 453         DPRINTF(("%s mapping Admin-CQ guest 0x%lx, host: %p\r\n",
 454                 __func__, sc->regs.acq, sc->compl_queues[0].qbase));
 455 }
 456
 457 static int
 458 nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
 459         struct nvme_completion* compl)
 460 {
 461         uint16_t qid = command->cdw10 & 0xffff;
 462
 463         DPRINTF(("%s DELETE_IO_SQ %u\r\n", __func__, qid));
 464         if (qid == 0 || qid > sc->num_squeues) {
 465                 WPRINTF(("%s NOT PERMITTED queue id %u / num_squeues %u\r\n",
 466                         __func__, qid, sc->num_squeues));
 467                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 468                     NVME_SC_INVALID_QUEUE_IDENTIFIER);
 469                 return (1);
 470         }
 471
 472         sc->submit_queues[qid].qbase = NULL;
 473         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 474         return (1);
 475 }
 476
 477 static int
 478 nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
 479         struct nvme_completion* compl)
 480 {
 481         if (command->cdw11 & NVME_CMD_CDW11_PC) {
 482                 uint16_t qid = command->cdw10 & 0xffff;
 483                 struct nvme_submission_queue *nsq;
 484
 485                 if ((qid == 0) || (qid > sc->num_squeues)) {
 486                         WPRINTF(("%s queue index %u > num_squeues %u\r\n",
 487                                 __func__, qid, sc->num_squeues));
 488                         pci_nvme_status_tc(&compl->status,
 489                             NVME_SCT_COMMAND_SPECIFIC,
 490                             NVME_SC_INVALID_QUEUE_IDENTIFIER);
 491                         return (1);
 492                 }
 493
 494                 nsq = &sc->submit_queues[qid];
 495                 nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
 496
 497                 nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 498                               sizeof(struct nvme_command) * (size_t)nsq->size);
 499                 nsq->cqid = (command->cdw11 >> 16) & 0xffff;
 500                 nsq->qpriority = (command->cdw11 >> 1) & 0x03;
 501
 502                 DPRINTF(("%s sq %u size %u gaddr %p cqid %u\r\n", __func__,
 503                         qid, nsq->size, nsq->qbase, nsq->cqid));
 504
 505                 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 506
 507                 DPRINTF(("%s completed creating IOSQ qid %u\r\n",
 508                          __func__, qid));
 509         } else {
 510                 /*
 511                  * Guest sent non-cont submission queue request.
 512                  * This setting is unsupported by this emulation.
 513                  */
 514                 WPRINTF(("%s unsupported non-contig (list-based) "
 515                          "create i/o submission queue\r\n", __func__));
 516
 517                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 518         }
 519         return (1);
 520 }
 521
 522 static int
 523 nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
 524         struct nvme_completion* compl)
 525 {
 526         uint16_t qid = command->cdw10 & 0xffff;
 527
 528         DPRINTF(("%s DELETE_IO_CQ %u\r\n", __func__, qid));
 529         if (qid == 0 || qid > sc->num_cqueues) {
 530                 WPRINTF(("%s queue index %u / num_cqueues %u\r\n",
 531                         __func__, qid, sc->num_cqueues));
 532                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 533                     NVME_SC_INVALID_QUEUE_IDENTIFIER);
 534                 return (1);
 535         }
 536
 537         sc->compl_queues[qid].qbase = NULL;
 538         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 539         return (1);
 540 }
 541
 542 static int
 543 nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
 544         struct nvme_completion* compl)
 545 {
 546         if (command->cdw11 & NVME_CMD_CDW11_PC) {
 547                 uint16_t qid = command->cdw10 & 0xffff;
 548                 struct nvme_completion_queue *ncq;
 549
 550                 if ((qid == 0) || (qid > sc->num_cqueues)) {
 551                         WPRINTF(("%s queue index %u > num_cqueues %u\r\n",
 552                                 __func__, qid, sc->num_cqueues));
 553                         pci_nvme_status_tc(&compl->status,
 554                             NVME_SCT_COMMAND_SPECIFIC,
 555                             NVME_SC_INVALID_QUEUE_IDENTIFIER);
 556                         return (1);
 557                 }
 558
 559                 ncq = &sc->compl_queues[qid];
 560                 ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1;
 561                 ncq->intr_vec = (command->cdw11 >> 16) & 0xffff;
 562                 ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
 563
 564                 ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
 565                              command->prp1,
 566                              sizeof(struct nvme_command) * (size_t)ncq->size);
 567
 568                 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 569         } else {
 570                 /*
 571                  * Non-contig completion queue unsupported.
 572                  */
 573                 WPRINTF(("%s unsupported non-contig (list-based) "
 574                          "create i/o completion queue\r\n",
 575                          __func__));
 576
 577                 /* 0x12 = Invalid Use of Controller Memory Buffer */
 578                 pci_nvme_status_genc(&compl->status, 0x12);
 579         }
 580
 581         return (1);
 582 }
 583
 584 static int
 585 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command,
 586         struct nvme_completion* compl)
 587 {
 588         uint32_t logsize = (1 + ((command->cdw10 >> 16) & 0xFFF)) * 2;
 589         uint8_t logpage = command->cdw10 & 0xFF;
 590         void *data;
 591
 592         DPRINTF(("%s log page %u len %u\r\n", __func__, logpage, logsize));
 593
 594         if (logpage >= 1 && logpage <= 3)
 595                 data = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 596                                   PAGE_SIZE);
 597
 598         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 599
 600         switch (logpage) {
 601         case 0x01: /* Error information */
 602                 memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
 603                 break;
 604         case 0x02: /* SMART/Health information */
 605                 /* TODO: present some smart info */
 606                 memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
 607                 break;
 608         case 0x03: /* Firmware slot information */
 609                 memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
 610                 break;
 611         default:
 612                 WPRINTF(("%s get log page %x command not supported\r\n",
 613                         __func__, logpage));
 614
 615                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 616                     NVME_SC_INVALID_LOG_PAGE);
 617         }
 618
 619         return (1);
 620 }
 621
 622 static int
 623 nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command,
 624         struct nvme_completion* compl)
 625 {
 626         void *dest;
 627
 628         DPRINTF(("%s identify 0x%x nsid 0x%x\r\n", __func__,
 629                 command->cdw10 & 0xFF, command->nsid));
 630
 631         switch (command->cdw10 & 0xFF) {
 632         case 0x00: /* return Identify Namespace data structure */
 633                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 634                                   sizeof(sc->nsdata));
 635                 memcpy(dest, &sc->nsdata, sizeof(sc->nsdata));
 636                 break;
 637         case 0x01: /* return Identify Controller data structure */
 638                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 639                                   sizeof(sc->ctrldata));
 640                 memcpy(dest, &sc->ctrldata, sizeof(sc->ctrldata));
 641                 break;
 642         case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */
 643                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 644                                   sizeof(uint32_t) * 1024);
 645                 ((uint32_t *)dest)[0] = 1;
 646                 ((uint32_t *)dest)[1] = 0;
 647                 break;
 648         case 0x11:
 649                 pci_nvme_status_genc(&compl->status,
 650                     NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
 651                 return (1);
 652         case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */
 653         case 0x10:
 654         case 0x12:
 655         case 0x13:
 656         case 0x14:
 657         case 0x15:
 658         default:
 659                 DPRINTF(("%s unsupported identify command requested 0x%x\r\n",
 660                          __func__, command->cdw10 & 0xFF));
 661                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 662                 return (1);
 663         }
 664
 665         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 666         return (1);
 667 }
 668
 669 static int
 670 nvme_set_feature_queues(struct pci_nvme_softc* sc, struct nvme_command* command,
 671         struct nvme_completion* compl)
 672 {
 673         uint16_t nqr;   /* Number of Queues Requested */
 674
 675         nqr = command->cdw11 & 0xFFFF;
 676         if (nqr == 0xffff) {
 677                 WPRINTF(("%s: Illegal NSQR value %#x\n", __func__, nqr));
 678                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 679                 return (-1);
 680         }
 681
 682         sc->num_squeues = ONE_BASED(nqr);
 683         if (sc->num_squeues > sc->max_queues) {
 684                 DPRINTF(("NSQR=%u is greater than max %u\n", sc->num_squeues,
 685                                         sc->max_queues));
 686                 sc->num_squeues = sc->max_queues;
 687         }
 688
 689         nqr = (command->cdw11 >> 16) & 0xFFFF;
 690         if (nqr == 0xffff) {
 691                 WPRINTF(("%s: Illegal NCQR value %#x\n", __func__, nqr));
 692                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 693                 return (-1);
 694         }
 695
 696         sc->num_cqueues = ONE_BASED(nqr);
 697         if (sc->num_cqueues > sc->max_queues) {
 698                 DPRINTF(("NCQR=%u is greater than max %u\n", sc->num_cqueues,
 699                                         sc->max_queues));
 700                 sc->num_cqueues = sc->max_queues;
 701         }
 702
 703         compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc);
 704
 705         return (0);
 706 }
 707
 708 static int
 709 nvme_opc_set_features(struct pci_nvme_softc* sc, struct nvme_command* command,
 710         struct nvme_completion* compl)
 711 {
 712         int feature = command->cdw10 & 0xFF;
 713         uint32_t iv;
 714
 715         DPRINTF(("%s feature 0x%x\r\n", __func__, feature));
 716         compl->cdw0 = 0;
 717
 718         switch (feature) {
 719         case NVME_FEAT_ARBITRATION:
 720                 DPRINTF(("  arbitration 0x%x\r\n", command->cdw11));
 721                 break;
 722         case NVME_FEAT_POWER_MANAGEMENT:
 723                 DPRINTF(("  power management 0x%x\r\n", command->cdw11));
 724                 break;
 725         case NVME_FEAT_LBA_RANGE_TYPE:
 726                 DPRINTF(("  lba range 0x%x\r\n", command->cdw11));
 727                 break;
 728         case NVME_FEAT_TEMPERATURE_THRESHOLD:
 729                 DPRINTF(("  temperature threshold 0x%x\r\n", command->cdw11));
 730                 break;
 731         case NVME_FEAT_ERROR_RECOVERY:
 732                 DPRINTF(("  error recovery 0x%x\r\n", command->cdw11));
 733                 break;
 734         case NVME_FEAT_VOLATILE_WRITE_CACHE:
 735                 DPRINTF(("  volatile write cache 0x%x\r\n", command->cdw11));
 736                 break;
 737         case NVME_FEAT_NUMBER_OF_QUEUES:
 738                 nvme_set_feature_queues(sc, command, compl);
 739                 break;
 740         case NVME_FEAT_INTERRUPT_COALESCING:
 741                 DPRINTF(("  interrupt coalescing 0x%x\r\n", command->cdw11));
 742
 743                 /* in uS */
 744                 sc->intr_coales_aggr_time = ((command->cdw11 >> 8) & 0xFF)*100;
 745
 746                 sc->intr_coales_aggr_thresh = command->cdw11 & 0xFF;
 747                 break;
 748         case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
 749                 iv = command->cdw11 & 0xFFFF;
 750
 751                 DPRINTF(("  interrupt vector configuration 0x%x\r\n",
 752                         command->cdw11));
 753
 754                 for (uint32_t i = 0; i < sc->num_cqueues + 1; i++) {
 755                         if (sc->compl_queues[i].intr_vec == iv) {
 756                                 if (command->cdw11 & (1 << 16))
 757                                         sc->compl_queues[i].intr_en |=
 758                                                               NVME_CQ_INTCOAL;
 759                                 else
 760                                         sc->compl_queues[i].intr_en &=
 761                                                              ~NVME_CQ_INTCOAL;
 762                         }
 763                 }
 764                 break;
 765         case NVME_FEAT_WRITE_ATOMICITY:
 766                 DPRINTF(("  write atomicity 0x%x\r\n", command->cdw11));
 767                 break;
 768         case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
 769                 DPRINTF(("  async event configuration 0x%x\r\n",
 770                         command->cdw11));
 771                 sc->async_ev_config = command->cdw11;
 772                 break;
 773         case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
 774                 DPRINTF(("  software progress marker 0x%x\r\n",
 775                         command->cdw11));
 776                 break;
 777         case 0x0C:
 778                 DPRINTF(("  autonomous power state transition 0x%x\r\n",
 779                         command->cdw11));
 780                 break;
 781         default:
 782                 WPRINTF(("%s invalid feature\r\n", __func__));
 783                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 784                 return (1);
 785         }
 786
 787         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 788         return (1);
 789 }
 790
 791 static int
 792 nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command,
 793         struct nvme_completion* compl)
 794 {
 795         int feature = command->cdw10 & 0xFF;
 796
 797         DPRINTF(("%s feature 0x%x\r\n", __func__, feature));
 798
 799         compl->cdw0 = 0;
 800
 801         switch (feature) {
 802         case NVME_FEAT_ARBITRATION:
 803                 DPRINTF(("  arbitration\r\n"));
 804                 break;
 805         case NVME_FEAT_POWER_MANAGEMENT:
 806                 DPRINTF(("  power management\r\n"));
 807                 break;
 808         case NVME_FEAT_LBA_RANGE_TYPE:
 809                 DPRINTF(("  lba range\r\n"));
 810                 break;
 811         case NVME_FEAT_TEMPERATURE_THRESHOLD:
 812                 DPRINTF(("  temperature threshold\r\n"));
 813                 switch ((command->cdw11 >> 20) & 0x3) {
 814                 case 0:
 815                         /* Over temp threshold */
 816                         compl->cdw0 = 0xFFFF;
 817                         break;
 818                 case 1:
 819                         /* Under temp threshold */
 820                         compl->cdw0 = 0;
 821                         break;
 822                 default:
 823                         WPRINTF(("  invalid threshold type select\r\n"));
 824                         pci_nvme_status_genc(&compl->status,
 825                             NVME_SC_INVALID_FIELD);
 826                         return (1);
 827                 }
 828                 break;
 829         case NVME_FEAT_ERROR_RECOVERY:
 830                 DPRINTF(("  error recovery\r\n"));
 831                 break;
 832         case NVME_FEAT_VOLATILE_WRITE_CACHE:
 833                 DPRINTF(("  volatile write cache\r\n"));
 834                 break;
 835         case NVME_FEAT_NUMBER_OF_QUEUES:
 836                 compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc);
 837
 838                 DPRINTF(("  number of queues (submit %u, completion %u)\r\n",
 839                         compl->cdw0 & 0xFFFF,
 840                         (compl->cdw0 >> 16) & 0xFFFF));
 841
 842                 break;
 843         case NVME_FEAT_INTERRUPT_COALESCING:
 844                 DPRINTF(("  interrupt coalescing\r\n"));
 845                 break;
 846         case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
 847                 DPRINTF(("  interrupt vector configuration\r\n"));
 848                 break;
 849         case NVME_FEAT_WRITE_ATOMICITY:
 850                 DPRINTF(("  write atomicity\r\n"));
 851                 break;
 852         case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
 853                 DPRINTF(("  async event configuration\r\n"));
 854                 sc->async_ev_config = command->cdw11;
 855                 break;
 856         case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
 857                 DPRINTF(("  software progress marker\r\n"));
 858                 break;
 859         case 0x0C:
 860                 DPRINTF(("  autonomous power state transition\r\n"));
 861                 break;
 862         default:
 863                 WPRINTF(("%s invalid feature 0x%x\r\n", __func__, feature));
 864                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 865                 return (1);
 866         }
 867
 868         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 869         return (1);
 870 }
 871
 872 static int
 873 nvme_opc_abort(struct pci_nvme_softc* sc, struct nvme_command* command,
 874         struct nvme_completion* compl)
 875 {
 876         DPRINTF(("%s submission queue %u, command ID 0x%x\r\n", __func__,
 877                 command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF));
 878
 879         /* TODO: search for the command ID and abort it */
 880
 881         compl->cdw0 = 1;
 882         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 883         return (1);
 884 }
 885
 886 static int
 887 nvme_opc_async_event_req(struct pci_nvme_softc* sc,
 888         struct nvme_command* command, struct nvme_completion* compl)
 889 {
 890         DPRINTF(("%s async event request 0x%x\r\n", __func__, command->cdw11));
 891
 892         /*
 893          * TODO: raise events when they happen based on the Set Features cmd.
 894          * These events happen async, so only set completion successful if
 895          * there is an event reflective of the request to get event.
 896          */
 897         pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 898             NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
 899         return (0);
 900 }
 901
 902 static void
 903 pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
 904 {
 905         struct nvme_completion compl;
 906         struct nvme_command *cmd;
 907         struct nvme_submission_queue *sq;
 908         struct nvme_completion_queue *cq;
 909         int do_intr = 0;
 910         uint16_t sqhead;
 911
 912         DPRINTF(("%s index %u\r\n", __func__, (uint32_t)value));
 913
 914         sq = &sc->submit_queues[0];
 915
 916         sqhead = atomic_load_acq_short(&sq->head);
 917
 918         if (atomic_testandset_int(&sq->busy, 1)) {
 919                 DPRINTF(("%s SQ busy, head %u, tail %u\r\n",
 920                         __func__, sqhead, sq->tail));
 921                 return;
 922         }
 923
 924         DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail));
 925
 926         while (sqhead != atomic_load_acq_short(&sq->tail)) {
 927                 cmd = &(sq->qbase)[sqhead];
 928                 compl.status = 0;
 929
 930                 switch (cmd->opc) {
 931                 case NVME_OPC_DELETE_IO_SQ:
 932                         DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__));
 933                         do_intr |= nvme_opc_delete_io_sq(sc, cmd, &compl);
 934                         break;
 935                 case NVME_OPC_CREATE_IO_SQ:
 936                         DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__));
 937                         do_intr |= nvme_opc_create_io_sq(sc, cmd, &compl);
 938                         break;
 939                 case NVME_OPC_DELETE_IO_CQ:
 940                         DPRINTF(("%s command DELETE_IO_CQ\r\n", __func__));
 941                         do_intr |= nvme_opc_delete_io_cq(sc, cmd, &compl);
 942                         break;
 943                 case NVME_OPC_CREATE_IO_CQ:
 944                         DPRINTF(("%s command CREATE_IO_CQ\r\n", __func__));
 945                         do_intr |= nvme_opc_create_io_cq(sc, cmd, &compl);
 946                         break;
 947                 case NVME_OPC_GET_LOG_PAGE:
 948                         DPRINTF(("%s command GET_LOG_PAGE\r\n", __func__));
 949                         do_intr |= nvme_opc_get_log_page(sc, cmd, &compl);
 950                         break;
 951                 case NVME_OPC_IDENTIFY:
 952                         DPRINTF(("%s command IDENTIFY\r\n", __func__));
 953                         do_intr |= nvme_opc_identify(sc, cmd, &compl);
 954                         break;
 955                 case NVME_OPC_ABORT:
 956                         DPRINTF(("%s command ABORT\r\n", __func__));
 957                         do_intr |= nvme_opc_abort(sc, cmd, &compl);
 958                         break;
 959                 case NVME_OPC_SET_FEATURES:
 960                         DPRINTF(("%s command SET_FEATURES\r\n", __func__));
 961                         do_intr |= nvme_opc_set_features(sc, cmd, &compl);
 962                         break;
 963                 case NVME_OPC_GET_FEATURES:
 964                         DPRINTF(("%s command GET_FEATURES\r\n", __func__));
 965                         do_intr |= nvme_opc_get_features(sc, cmd, &compl);
 966                         break;
 967                 case NVME_OPC_ASYNC_EVENT_REQUEST:
 968                         DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__));
 969                         /* XXX dont care, unhandled for now
 970                         do_intr |= nvme_opc_async_event_req(sc, cmd, &compl);
 971                         */
 972                         break;
 973                 default:
 974                         WPRINTF(("0x%x command is not implemented\r\n",
 975                             cmd->opc));
 976                 }
 977
 978                 /* for now skip async event generation */
 979                 if (cmd->opc != NVME_OPC_ASYNC_EVENT_REQUEST) {
 980                         struct nvme_completion *cp;
 981                         int phase;
 982
 983                         cq = &sc->compl_queues[0];
 984
 985                         cp = &(cq->qbase)[cq->tail];
 986                         cp->cdw0 = compl.cdw0;
 987                         cp->sqid = 0;
 988                         cp->sqhd = sqhead;
 989                         cp->cid = cmd->cid;
 990
 991                         phase = NVME_STATUS_GET_P(cp->status);
 992                         cp->status = compl.status;
 993                         pci_nvme_toggle_phase(&cp->status, phase);
 994
 995                         cq->tail = (cq->tail + 1) % cq->size;
 996                 }
 997                 sqhead = (sqhead + 1) % sq->size;
 998         }
 999
1000         DPRINTF(("setting sqhead %u\r\n", sqhead));
1001         atomic_store_short(&sq->head, sqhead);
1002         atomic_store_int(&sq->busy, 0);
1003
1004         if (do_intr)
1005                 pci_generate_msix(sc->nsc_pi, 0);
1006
1007 }
1008
1009 static int
1010 pci_nvme_append_iov_req(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req,
1011         uint64_t gpaddr, size_t size, int do_write, uint64_t lba)
1012 {
1013         int iovidx;
1014
1015         if (req != NULL) {
1016                 /* concatenate contig block-iovs to minimize number of iovs */
1017                 if ((req->prev_gpaddr + req->prev_size) == gpaddr) {
1018                         iovidx = req->io_req.br_iovcnt - 1;
1019
1020                         req->io_req.br_iov[iovidx].iov_base =
1021                             paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
1022                                              req->prev_gpaddr, size);
1023
1024                         req->prev_size += size;
1025                         req->io_req.br_resid += size;
1026
1027                         req->io_req.br_iov[iovidx].iov_len = req->prev_size;
1028                 } else {
1029                         pthread_mutex_lock(&req->mtx);
1030
1031                         iovidx = req->io_req.br_iovcnt;
1032                         if (iovidx == NVME_MAX_BLOCKIOVS) {
1033                                 int err = 0;
1034
1035                                 DPRINTF(("large I/O, doing partial req\r\n"));
1036
1037                                 iovidx = 0;
1038                                 req->io_req.br_iovcnt = 0;
1039
1040                                 req->io_req.br_callback = pci_nvme_io_partial;
1041
1042                                 if (!do_write)
1043                                         err = blockif_read(sc->nvstore.ctx,
1044                                                            &req->io_req);
1045                                 else
1046                                         err = blockif_write(sc->nvstore.ctx,
1047                                                             &req->io_req);
1048
1049                                 /* wait until req completes before cont */
1050                                 if (err == 0)
1051                                         pthread_cond_wait(&req->cv, &req->mtx);
1052                         }
1053                         if (iovidx == 0) {
1054                                 req->io_req.br_offset = lba;
1055                                 req->io_req.br_resid = 0;
1056                                 req->io_req.br_param = req;
1057                         }
1058
1059                         req->io_req.br_iov[iovidx].iov_base =
1060                             paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
1061                                              gpaddr, size);
1062
1063                         req->io_req.br_iov[iovidx].iov_len = size;
1064
1065                         req->prev_gpaddr = gpaddr;
1066                         req->prev_size = size;
1067                         req->io_req.br_resid += size;
1068
1069                         req->io_req.br_iovcnt++;
1070
1071                         pthread_mutex_unlock(&req->mtx);
1072                 }
1073         } else {
1074                 /* RAM buffer: read/write directly */
1075                 void *p = sc->nvstore.ctx;
1076                 void *gptr;
1077
1078                 if ((lba + size) > sc->nvstore.size) {
1079                         WPRINTF(("%s write would overflow RAM\r\n", __func__));
1080                         return (-1);
1081                 }
1082
1083                 p = (void *)((uintptr_t)p + (uintptr_t)lba);
1084                 gptr = paddr_guest2host(sc->nsc_pi->pi_vmctx, gpaddr, size);
1085                 if (do_write)
1086                         memcpy(p, gptr, size);
1087                 else
1088                         memcpy(gptr, p, size);
1089         }
1090         return (0);
1091 }
1092
1093 static void
1094 pci_nvme_set_completion(struct pci_nvme_softc *sc,
1095         struct nvme_submission_queue *sq, int sqid, uint16_t cid,
1096         uint32_t cdw0, uint16_t status, int ignore_busy)
1097 {
1098         struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
1099         struct nvme_completion *compl;
1100         int do_intr = 0;
1101         int phase;
1102
1103         DPRINTF(("%s sqid %d cqid %u cid %u status: 0x%x 0x%x\r\n",
1104                  __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
1105                  NVME_STATUS_GET_SC(status)));
1106
1107         pthread_mutex_lock(&cq->mtx);
1108
1109         assert(cq->qbase != NULL);
1110
1111         compl = &cq->qbase[cq->tail];
1112
1113         compl->sqhd = atomic_load_acq_short(&sq->head);
1114         compl->sqid = sqid;
1115         compl->cid = cid;
1116
1117         // toggle phase
1118         phase = NVME_STATUS_GET_P(compl->status);
1119         compl->status = status;
1120         pci_nvme_toggle_phase(&compl->status, phase);
1121
1122         cq->tail = (cq->tail + 1) % cq->size;
1123
1124         if (cq->intr_en & NVME_CQ_INTEN)
1125                 do_intr = 1;
1126
1127         pthread_mutex_unlock(&cq->mtx);
1128
1129         if (ignore_busy || !atomic_load_acq_int(&sq->busy))
1130                 if (do_intr)
1131                         pci_generate_msix(sc->nsc_pi, cq->intr_vec);
1132 }
1133
1134 static void
1135 pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req)
1136 {
1137         req->sc = NULL;
1138         req->nvme_sq = NULL;
1139         req->sqid = 0;
1140
1141         pthread_mutex_lock(&sc->mtx);
1142
1143         req->next = sc->ioreqs_free;
1144         sc->ioreqs_free = req;
1145         sc->pending_ios--;
1146
1147         /* when no more IO pending, can set to ready if device reset/enabled */
1148         if (sc->pending_ios == 0 &&
1149             NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts)))
1150                 sc->regs.csts |= NVME_CSTS_RDY;
1151
1152         pthread_mutex_unlock(&sc->mtx);
1153
1154         sem_post(&sc->iosemlock);
1155 }
1156
1157 static struct pci_nvme_ioreq *
1158 pci_nvme_get_ioreq(struct pci_nvme_softc *sc)
1159 {
1160         struct pci_nvme_ioreq *req = NULL;;
1161
1162         sem_wait(&sc->iosemlock);
1163         pthread_mutex_lock(&sc->mtx);
1164
1165         req = sc->ioreqs_free;
1166         assert(req != NULL);
1167
1168         sc->ioreqs_free = req->next;
1169
1170         req->next = NULL;
1171         req->sc = sc;
1172
1173         sc->pending_ios++;
1174
1175         pthread_mutex_unlock(&sc->mtx);
1176
1177         req->io_req.br_iovcnt = 0;
1178         req->io_req.br_offset = 0;
1179         req->io_req.br_resid = 0;
1180         req->io_req.br_param = req;
1181         req->prev_gpaddr = 0;
1182         req->prev_size = 0;
1183
1184         return req;
1185 }
1186
1187 static void
1188 pci_nvme_io_done(struct blockif_req *br, int err)
1189 {
1190         struct pci_nvme_ioreq *req = br->br_param;
1191         struct nvme_submission_queue *sq = req->nvme_sq;
1192         uint16_t code, status;
1193
1194         DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err)));
1195
1196         /* TODO return correct error */
1197         code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;
1198         pci_nvme_status_genc(&status, code);
1199
1200         pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status, 0);
1201         pci_nvme_release_ioreq(req->sc, req);
1202 }
1203
1204 static void
1205 pci_nvme_io_partial(struct blockif_req *br, int err)
1206 {
1207         struct pci_nvme_ioreq *req = br->br_param;
1208
1209         DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err)));
1210
1211         pthread_cond_signal(&req->cv);
1212 }
1213
1214
1215 static void
1216 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx)
1217 {
1218         struct nvme_submission_queue *sq;
1219         uint16_t status;
1220         uint16_t sqhead;
1221         int err;
1222
1223         /* handle all submissions up to sq->tail index */
1224         sq = &sc->submit_queues[idx];
1225
1226         if (atomic_testandset_int(&sq->busy, 1)) {
1227                 DPRINTF(("%s sqid %u busy\r\n", __func__, idx));
1228                 return;
1229         }
1230
1231         sqhead = atomic_load_acq_short(&sq->head);
1232
1233         DPRINTF(("nvme_handle_io qid %u head %u tail %u cmdlist %p\r\n",
1234                  idx, sqhead, sq->tail, sq->qbase));
1235
1236         while (sqhead != atomic_load_acq_short(&sq->tail)) {
1237                 struct nvme_command *cmd;
1238                 struct pci_nvme_ioreq *req = NULL;
1239                 uint64_t lba;
1240                 uint64_t nblocks, bytes, size, cpsz;
1241
1242                 /* TODO: support scatter gather list handling */
1243
1244                 cmd = &sq->qbase[sqhead];
1245                 sqhead = (sqhead + 1) % sq->size;
1246
1247                 lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
1248
1249                 if (cmd->opc == NVME_OPC_FLUSH) {
1250                         pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1251                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1252                                                 status, 1);
1253
1254                         continue;
1255                 } else if (cmd->opc == 0x08) {
1256                         /* TODO: write zeroes */
1257                         WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n",
1258                                 __func__, lba, cmd->cdw12 & 0xFFFF));
1259                         pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1260                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1261                                                 status, 1);
1262
1263                         continue;
1264                 }
1265
1266                 nblocks = (cmd->cdw12 & 0xFFFF) + 1;
1267
1268                 bytes = nblocks * sc->nvstore.sectsz;
1269
1270                 if (sc->nvstore.type == NVME_STOR_BLOCKIF) {
1271                         req = pci_nvme_get_ioreq(sc);
1272                         req->nvme_sq = sq;
1273                         req->sqid = idx;
1274                 }
1275
1276                 /*
1277                  * If data starts mid-page and flows into the next page, then
1278                  * increase page count
1279                  */
1280
1281                 DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu "
1282                          "(%lu-bytes)\r\n",
1283                          sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size,
1284                          cmd->opc == NVME_OPC_WRITE ?
1285                              "WRITE" : "READ",
1286                          lba, nblocks, bytes));
1287
1288                 cmd->prp1 &= ~(0x03UL);
1289                 cmd->prp2 &= ~(0x03UL);
1290
1291                 DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2));
1292
1293                 size = bytes;
1294                 lba *= sc->nvstore.sectsz;
1295
1296                 cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE);
1297
1298                 if (cpsz > bytes)
1299                         cpsz = bytes;
1300
1301                 if (req != NULL) {
1302                         req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) |
1303                                                 cmd->cdw10;
1304                         req->opc = cmd->opc;
1305                         req->cid = cmd->cid;
1306                         req->nsid = cmd->nsid;
1307                 }
1308
1309                 err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz,
1310                     cmd->opc == NVME_OPC_WRITE, lba);
1311                 lba += cpsz;
1312                 size -= cpsz;
1313
1314                 if (size == 0)
1315                         goto iodone;
1316
1317                 if (size <= PAGE_SIZE) {
1318                         /* prp2 is second (and final) page in transfer */
1319
1320                         err = pci_nvme_append_iov_req(sc, req, cmd->prp2,
1321                             size,
1322                             cmd->opc == NVME_OPC_WRITE,
1323                             lba);
1324                 } else {
1325                         uint64_t *prp_list;
1326                         int i;
1327
1328                         /* prp2 is pointer to a physical region page list */
1329                         prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx,
1330                                                     cmd->prp2, PAGE_SIZE);
1331
1332                         i = 0;
1333                         while (size != 0) {
1334                                 cpsz = MIN(size, PAGE_SIZE);
1335
1336                                 /*
1337                                  * Move to linked physical region page list
1338                                  * in last item.
1339                                  */
1340                                 if (i == (NVME_PRP2_ITEMS-1) &&
1341                                     size > PAGE_SIZE) {
1342                                         assert((prp_list[i] & (PAGE_SIZE-1)) == 0);
1343                                         prp_list = paddr_guest2host(
1344                                                       sc->nsc_pi->pi_vmctx,
1345                                                       prp_list[i], PAGE_SIZE);
1346                                         i = 0;
1347                                 }
1348                                 if (prp_list[i] == 0) {
1349                                         WPRINTF(("PRP2[%d] = 0 !!!\r\n", i));
1350                                         err = 1;
1351                                         break;
1352                                 }
1353
1354                                 err = pci_nvme_append_iov_req(sc, req,
1355                                     prp_list[i], cpsz,
1356                                     cmd->opc == NVME_OPC_WRITE, lba);
1357                                 if (err)
1358                                         break;
1359
1360                                 lba += cpsz;
1361                                 size -= cpsz;
1362                                 i++;
1363                         }
1364                 }
1365
1366 iodone:
1367                 if (sc->nvstore.type == NVME_STOR_RAM) {
1368                         uint16_t code, status;
1369
1370                         code = err ? NVME_SC_LBA_OUT_OF_RANGE :
1371                             NVME_SC_SUCCESS;
1372                         pci_nvme_status_genc(&status, code);
1373
1374                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1375                                                 status, 1);
1376
1377                         continue;
1378                 }
1379
1380
1381                 if (err)
1382                         goto do_error;
1383
1384                 req->io_req.br_callback = pci_nvme_io_done;
1385
1386                 err = 0;
1387                 switch (cmd->opc) {
1388                 case NVME_OPC_READ:
1389                         err = blockif_read(sc->nvstore.ctx, &req->io_req);
1390                         break;
1391                 case NVME_OPC_WRITE:
1392                         err = blockif_write(sc->nvstore.ctx, &req->io_req);
1393                         break;
1394                 default:
1395                         WPRINTF(("%s unhandled io command 0x%x\r\n",
1396                                  __func__, cmd->opc));
1397                         err = 1;
1398                 }
1399
1400 do_error:
1401                 if (err) {
1402                         uint16_t status;
1403
1404                         pci_nvme_status_genc(&status,
1405                             NVME_SC_DATA_TRANSFER_ERROR);
1406
1407                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1408                                                 status, 1);
1409                         pci_nvme_release_ioreq(sc, req);
1410                 }
1411         }
1412
1413         atomic_store_short(&sq->head, sqhead);
1414         atomic_store_int(&sq->busy, 0);
1415 }
1416
1417 static void
1418 pci_nvme_handle_doorbell(struct vmctx *ctx, struct pci_nvme_softc* sc,
1419         uint64_t idx, int is_sq, uint64_t value)
1420 {
1421         DPRINTF(("nvme doorbell %lu, %s, val 0x%lx\r\n",
1422                 idx, is_sq ? "SQ" : "CQ", value & 0xFFFF));
1423
1424         if (is_sq) {
1425                 atomic_store_short(&sc->submit_queues[idx].tail,
1426                                    (uint16_t)value);
1427
1428                 if (idx == 0) {
1429                         pci_nvme_handle_admin_cmd(sc, value);
1430                 } else {
1431                         /* submission queue; handle new entries in SQ */
1432                         if (idx > sc->num_squeues) {
1433                                 WPRINTF(("%s SQ index %lu overflow from "
1434                                          "guest (max %u)\r\n",
1435                                          __func__, idx, sc->num_squeues));
1436                                 return;
1437                         }
1438                         pci_nvme_handle_io_cmd(sc, (uint16_t)idx);
1439                 }
1440         } else {
1441                 if (idx > sc->num_cqueues) {
1442                         WPRINTF(("%s queue index %lu overflow from "
1443                                  "guest (max %u)\r\n",
1444                                  __func__, idx, sc->num_cqueues));
1445                         return;
1446                 }
1447
1448                 sc->compl_queues[idx].head = (uint16_t)value;
1449         }
1450 }
1451
1452 static void
1453 pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)
1454 {
1455         const char *s = iswrite ? "WRITE" : "READ";
1456
1457         switch (offset) {
1458         case NVME_CR_CAP_LOW:
1459                 DPRINTF(("%s %s NVME_CR_CAP_LOW\r\n", func, s));
1460                 break;
1461         case NVME_CR_CAP_HI:
1462                 DPRINTF(("%s %s NVME_CR_CAP_HI\r\n", func, s));
1463                 break;
1464         case NVME_CR_VS:
1465                 DPRINTF(("%s %s NVME_CR_VS\r\n", func, s));
1466                 break;
1467         case NVME_CR_INTMS:
1468                 DPRINTF(("%s %s NVME_CR_INTMS\r\n", func, s));
1469                 break;
1470         case NVME_CR_INTMC:
1471                 DPRINTF(("%s %s NVME_CR_INTMC\r\n", func, s));
1472                 break;
1473         case NVME_CR_CC:
1474                 DPRINTF(("%s %s NVME_CR_CC\r\n", func, s));
1475                 break;
1476         case NVME_CR_CSTS:
1477                 DPRINTF(("%s %s NVME_CR_CSTS\r\n", func, s));
1478                 break;
1479         case NVME_CR_NSSR:
1480                 DPRINTF(("%s %s NVME_CR_NSSR\r\n", func, s));
1481                 break;
1482         case NVME_CR_AQA:
1483                 DPRINTF(("%s %s NVME_CR_AQA\r\n", func, s));
1484                 break;
1485         case NVME_CR_ASQ_LOW:
1486                 DPRINTF(("%s %s NVME_CR_ASQ_LOW\r\n", func, s));
1487                 break;
1488         case NVME_CR_ASQ_HI:
1489                 DPRINTF(("%s %s NVME_CR_ASQ_HI\r\n", func, s));
1490                 break;
1491         case NVME_CR_ACQ_LOW:
1492                 DPRINTF(("%s %s NVME_CR_ACQ_LOW\r\n", func, s));
1493                 break;
1494         case NVME_CR_ACQ_HI:
1495                 DPRINTF(("%s %s NVME_CR_ACQ_HI\r\n", func, s));
1496                 break;
1497         default:
1498                 DPRINTF(("unknown nvme bar-0 offset 0x%lx\r\n", offset));
1499         }
1500
1501 }
1502
1503 static void
1504 pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc,
1505         uint64_t offset, int size, uint64_t value)
1506 {
1507         uint32_t ccreg;
1508
1509         if (offset >= NVME_DOORBELL_OFFSET) {
1510                 uint64_t belloffset = offset - NVME_DOORBELL_OFFSET;
1511                 uint64_t idx = belloffset / 8; /* door bell size = 2*int */
1512                 int is_sq = (belloffset % 8) < 4;
1513
1514                 if (belloffset > ((sc->max_queues+1) * 8 - 4)) {
1515                         WPRINTF(("guest attempted an overflow write offset "
1516                                  "0x%lx, val 0x%lx in %s",
1517                                  offset, value, __func__));
1518                         return;
1519                 }
1520
1521                 pci_nvme_handle_doorbell(ctx, sc, idx, is_sq, value);
1522                 return;
1523         }
1524
1525         DPRINTF(("nvme-write offset 0x%lx, size %d, value 0x%lx\r\n",
1526                 offset, size, value));
1527
1528         if (size != 4) {
1529                 WPRINTF(("guest wrote invalid size %d (offset 0x%lx, "
1530                          "val 0x%lx) to bar0 in %s",
1531                          size, offset, value, __func__));
1532                 /* TODO: shutdown device */
1533                 return;
1534         }
1535
1536         pci_nvme_bar0_reg_dumps(__func__, offset, 1);
1537
1538         pthread_mutex_lock(&sc->mtx);
1539
1540         switch (offset) {
1541         case NVME_CR_CAP_LOW:
1542         case NVME_CR_CAP_HI:
1543                 /* readonly */
1544                 break;
1545         case NVME_CR_VS:
1546                 /* readonly */
1547                 break;
1548         case NVME_CR_INTMS:
1549                 /* MSI-X, so ignore */
1550                 break;
1551         case NVME_CR_INTMC:
1552                 /* MSI-X, so ignore */
1553                 break;
1554         case NVME_CR_CC:
1555                 ccreg = (uint32_t)value;
1556
1557                 DPRINTF(("%s NVME_CR_CC en %x css %x shn %x iosqes %u "
1558                          "iocqes %u\r\n",
1559                         __func__,
1560                          NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg),
1561                          NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg),
1562                          NVME_CC_GET_IOCQES(ccreg)));
1563
1564                 if (NVME_CC_GET_SHN(ccreg)) {
1565                         /* perform shutdown - flush out data to backend */
1566                         sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK <<
1567                             NVME_CSTS_REG_SHST_SHIFT);
1568                         sc->regs.csts |= NVME_SHST_COMPLETE <<
1569                             NVME_CSTS_REG_SHST_SHIFT;
1570                 }
1571                 if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) {
1572                         if (NVME_CC_GET_EN(ccreg) == 0)
1573                                 /* transition 1-> causes controller reset */
1574                                 pci_nvme_reset_locked(sc);
1575                         else
1576                                 pci_nvme_init_controller(ctx, sc);
1577                 }
1578
1579                 /* Insert the iocqes, iosqes and en bits from the write */
1580                 sc->regs.cc &= ~NVME_CC_WRITE_MASK;
1581                 sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK;
1582                 if (NVME_CC_GET_EN(ccreg) == 0) {
1583                         /* Insert the ams, mps and css bit fields */
1584                         sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK;
1585                         sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK;
1586                         sc->regs.csts &= ~NVME_CSTS_RDY;
1587                 } else if (sc->pending_ios == 0) {
1588                         sc->regs.csts |= NVME_CSTS_RDY;
1589                 }
1590                 break;
1591         case NVME_CR_CSTS:
1592                 break;
1593         case NVME_CR_NSSR:
1594                 /* ignore writes; don't support subsystem reset */
1595                 break;
1596         case NVME_CR_AQA:
1597                 sc->regs.aqa = (uint32_t)value;
1598                 break;
1599         case NVME_CR_ASQ_LOW:
1600                 sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) |
1601                                (0xFFFFF000 & value);
1602                 break;
1603         case NVME_CR_ASQ_HI:
1604                 sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) |
1605                                (value << 32);
1606                 break;
1607         case NVME_CR_ACQ_LOW:
1608                 sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) |
1609                                (0xFFFFF000 & value);
1610                 break;
1611         case NVME_CR_ACQ_HI:
1612                 sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) |
1613                                (value << 32);
1614                 break;
1615         default:
1616                 DPRINTF(("%s unknown offset 0x%lx, value 0x%lx size %d\r\n",
1617                          __func__, offset, value, size));
1618         }
1619         pthread_mutex_unlock(&sc->mtx);
1620 }
1621
1622 static void
1623 pci_nvme_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
1624                 int baridx, uint64_t offset, int size, uint64_t value)
1625 {
1626         struct pci_nvme_softc* sc = pi->pi_arg;
1627
1628         if (baridx == pci_msix_table_bar(pi) ||
1629             baridx == pci_msix_pba_bar(pi)) {
1630                 DPRINTF(("nvme-write baridx %d, msix: off 0x%lx, size %d, "
1631                          " value 0x%lx\r\n", baridx, offset, size, value));
1632
1633                 pci_emul_msix_twrite(pi, offset, size, value);
1634                 return;
1635         }
1636
1637         switch (baridx) {
1638         case 0:
1639                 pci_nvme_write_bar_0(ctx, sc, offset, size, value);
1640                 break;
1641
1642         default:
1643                 DPRINTF(("%s unknown baridx %d, val 0x%lx\r\n",
1644                          __func__, baridx, value));
1645         }
1646 }
1647
1648 static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc,
1649         uint64_t offset, int size)
1650 {
1651         uint64_t value;
1652
1653         pci_nvme_bar0_reg_dumps(__func__, offset, 0);
1654
1655         if (offset < NVME_DOORBELL_OFFSET) {
1656                 void *p = &(sc->regs);
1657                 pthread_mutex_lock(&sc->mtx);
1658                 memcpy(&value, (void *)((uintptr_t)p + offset), size);
1659                 pthread_mutex_unlock(&sc->mtx);
1660         } else {
1661                 value = 0;
1662                 WPRINTF(("pci_nvme: read invalid offset %ld\r\n", offset));
1663         }
1664
1665         switch (size) {
1666         case 1:
1667                 value &= 0xFF;
1668                 break;
1669         case 2:
1670                 value &= 0xFFFF;
1671                 break;
1672         case 4:
1673                 value &= 0xFFFFFFFF;
1674                 break;
1675         }
1676
1677         DPRINTF(("   nvme-read offset 0x%lx, size %d -> value 0x%x\r\n",
1678                  offset, size, (uint32_t)value));
1679
1680         return (value);
1681 }
1682
1683
1684
1685 static uint64_t
1686 pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx,
1687     uint64_t offset, int size)
1688 {
1689         struct pci_nvme_softc* sc = pi->pi_arg;
1690
1691         if (baridx == pci_msix_table_bar(pi) ||
1692             baridx == pci_msix_pba_bar(pi)) {
1693                 DPRINTF(("nvme-read bar: %d, msix: regoff 0x%lx, size %d\r\n",
1694                         baridx, offset, size));
1695
1696                 return pci_emul_msix_tread(pi, offset, size);
1697         }
1698
1699         switch (baridx) {
1700         case 0:
1701                 return pci_nvme_read_bar_0(sc, offset, size);
1702
1703         default:
1704                 DPRINTF(("unknown bar %d, 0x%lx\r\n", baridx, offset));
1705         }
1706
1707         return (0);
1708 }
1709
1710
1711 static int
1712 pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts)
1713 {
1714         char bident[sizeof("XX:X:X")];
1715         char    *uopt, *xopts, *config;
1716         uint32_t sectsz;
1717         int optidx;
1718
1719         sc->max_queues = NVME_QUEUES;
1720         sc->max_qentries = NVME_MAX_QENTRIES;
1721         sc->ioslots = NVME_IOSLOTS;
1722         sc->num_squeues = sc->max_queues;
1723         sc->num_cqueues = sc->max_queues;
1724         sectsz = 0;
1725
1726         uopt = strdup(opts);
1727         optidx = 0;
1728         snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
1729                  "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
1730         for (xopts = strtok(uopt, ",");
1731              xopts != NULL;
1732              xopts = strtok(NULL, ",")) {
1733
1734                 if ((config = strchr(xopts, '=')) != NULL)
1735                         *config++ = '\0';
1736
1737                 if (!strcmp("maxq", xopts)) {
1738                         sc->max_queues = atoi(config);
1739                 } else if (!strcmp("qsz", xopts)) {
1740                         sc->max_qentries = atoi(config);
1741                 } else if (!strcmp("ioslots", xopts)) {
1742                         sc->ioslots = atoi(config);
1743                 } else if (!strcmp("sectsz", xopts)) {
1744                         sectsz = atoi(config);
1745                 } else if (!strcmp("ser", xopts)) {
1746                         /*
1747                          * This field indicates the Product Serial Number in
1748                          * 7-bit ASCII, unused bytes should be space characters.
1749                          * Ref: NVMe v1.3c.
1750                          */
1751                         cpywithpad((char *)sc->ctrldata.sn,
1752                                    sizeof(sc->ctrldata.sn), config, ' ');
1753                 } else if (!strcmp("ram", xopts)) {
1754                         uint64_t sz = strtoull(&xopts[4], NULL, 10);
1755
1756                         sc->nvstore.type = NVME_STOR_RAM;
1757                         sc->nvstore.size = sz * 1024 * 1024;
1758                         sc->nvstore.ctx = calloc(1, sc->nvstore.size);
1759                         sc->nvstore.sectsz = 4096;
1760                         sc->nvstore.sectsz_bits = 12;
1761                         if (sc->nvstore.ctx == NULL) {
1762                                 perror("Unable to allocate RAM");
1763                                 free(uopt);
1764                                 return (-1);
1765                         }
1766                 } else if (optidx == 0) {
1767                         snprintf(bident, sizeof(bident), "%d:%d",
1768                                  sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
1769                         sc->nvstore.ctx = blockif_open(xopts, bident);
1770                         if (sc->nvstore.ctx == NULL) {
1771                                 perror("Could not open backing file");
1772                                 free(uopt);
1773                                 return (-1);
1774                         }
1775                         sc->nvstore.type = NVME_STOR_BLOCKIF;
1776                         sc->nvstore.size = blockif_size(sc->nvstore.ctx);
1777                 } else {
1778                         fprintf(stderr, "Invalid option %s\n", xopts);
1779                         free(uopt);
1780                         return (-1);
1781                 }
1782
1783                 optidx++;
1784         }
1785         free(uopt);
1786
1787         if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) {
1788                 fprintf(stderr, "backing store not specified\n");
1789                 return (-1);
1790         }
1791         if (sectsz == 512 || sectsz == 4096 || sectsz == 8192)
1792                 sc->nvstore.sectsz = sectsz;
1793         else if (sc->nvstore.type != NVME_STOR_RAM)
1794                 sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx);
1795         for (sc->nvstore.sectsz_bits = 9;
1796              (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;
1797              sc->nvstore.sectsz_bits++);
1798
1799         if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES)
1800                 sc->max_queues = NVME_QUEUES;
1801
1802         if (sc->max_qentries <= 0) {
1803                 fprintf(stderr, "Invalid qsz option\n");
1804                 return (-1);
1805         }
1806         if (sc->ioslots <= 0) {
1807                 fprintf(stderr, "Invalid ioslots option\n");
1808                 return (-1);
1809         }
1810
1811         return (0);
1812 }
1813
1814 static int
1815 pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
1816 {
1817         struct pci_nvme_softc *sc;
1818         uint32_t pci_membar_sz;
1819         int     error;
1820
1821         error = 0;
1822
1823         sc = calloc(1, sizeof(struct pci_nvme_softc));
1824         pi->pi_arg = sc;
1825         sc->nsc_pi = pi;
1826
1827         error = pci_nvme_parse_opts(sc, opts);
1828         if (error < 0)
1829                 goto done;
1830         else
1831                 error = 0;
1832
1833         sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq));
1834         for (int i = 0; i < sc->ioslots; i++) {
1835                 if (i < (sc->ioslots-1))
1836                         sc->ioreqs[i].next = &sc->ioreqs[i+1];
1837                 pthread_mutex_init(&sc->ioreqs[i].mtx, NULL);
1838                 pthread_cond_init(&sc->ioreqs[i].cv, NULL);
1839         }
1840         sc->ioreqs_free = sc->ioreqs;
1841         sc->intr_coales_aggr_thresh = 1;
1842
1843         pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A);
1844         pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D);
1845         pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
1846         pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM);
1847         pci_set_cfgdata8(pi, PCIR_PROGIF,
1848                          PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0);
1849
1850         /* allocate size of nvme registers + doorbell space for all queues */
1851         pci_membar_sz = sizeof(struct nvme_registers) +
1852                         2*sizeof(uint32_t)*(sc->max_queues + 1);
1853
1854         DPRINTF(("nvme membar size: %u\r\n", pci_membar_sz));
1855
1856         error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz);
1857         if (error) {
1858                 WPRINTF(("%s pci alloc mem bar failed\r\n", __func__));
1859                 goto done;
1860         }
1861
1862         error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR);
1863         if (error) {
1864                 WPRINTF(("%s pci add msixcap failed\r\n", __func__));
1865                 goto done;
1866         }
1867
1868         pthread_mutex_init(&sc->mtx, NULL);
1869         sem_init(&sc->iosemlock, 0, sc->ioslots);
1870
1871         pci_nvme_reset(sc);
1872         pci_nvme_init_ctrldata(sc);
1873         pci_nvme_init_nsdata(sc);
1874
1875         pci_lintr_request(pi);
1876
1877 done:
1878         return (error);
1879 }
1880
1881
1882 struct pci_devemu pci_de_nvme = {
1883         .pe_emu =       "nvme",
1884         .pe_init =      pci_nvme_init,
1885         .pe_barwrite =  pci_nvme_write,
1886         .pe_barread =   pci_nvme_read
1887 };
1888 PCI_EMUL_SET(pci_de_nvme);