usr.sbin/bhyve/pci_nvme.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2017 Shunsuke Mie
   5  * Copyright (c) 2018 Leon Dang
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * bhyve PCIe-NVMe device emulation.
  31  *
  32  * options:
  33  *  -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z
  34  *
  35  *  accepted devpath:
  36  *    /dev/blockdev
  37  *    /path/to/image
  38  *    ram=size_in_MiB
  39  *
  40  *  maxq    = max number of queues
  41  *  qsz     = max elements in each queue
  42  *  ioslots = max number of concurrent io requests
  43  *  sectsz  = sector size (defaults to blockif sector size)
  44  *  ser     = serial number (20-chars max)
  45  *
  46  */
  47
  48 /* TODO:
  49     - create async event for smart and log
  50     - intr coalesce
  51  */
  52
  53 #include <sys/cdefs.h>
  54 __FBSDID("$FreeBSD$");
  55
  56 #include <sys/types.h>
  57
  58 #include <assert.h>
  59 #include <pthread.h>
  60 #include <semaphore.h>
  61 #include <stdbool.h>
  62 #include <stddef.h>
  63 #include <stdint.h>
  64 #include <stdio.h>
  65 #include <stdlib.h>
  66 #include <string.h>
  67
  68 #include <machine/atomic.h>
  69 #include <machine/vmm.h>
  70 #include <vmmapi.h>
  71
  72 #include <dev/nvme/nvme.h>
  73
  74 #include "bhyverun.h"
  75 #include "block_if.h"
  76 #include "pci_emul.h"
  77
  78
  79 static int nvme_debug = 0;
  80 #define DPRINTF(params) if (nvme_debug) printf params
  81 #define WPRINTF(params) printf params
  82
  83 /* defaults; can be overridden */
  84 #define NVME_MSIX_BAR           4
  85
  86 #define NVME_IOSLOTS            8
  87
  88 #define NVME_QUEUES             16
  89 #define NVME_MAX_QENTRIES       2048
  90
  91 #define NVME_PRP2_ITEMS         (PAGE_SIZE/sizeof(uint64_t))
  92 #define NVME_MAX_BLOCKIOVS      512
  93
  94 /* helpers */
  95
  96 #define NVME_DOORBELL_OFFSET    offsetof(struct nvme_registers, doorbell)
  97
  98 enum nvme_controller_register_offsets {
  99         NVME_CR_CAP_LOW = 0x00,
 100         NVME_CR_CAP_HI  = 0x04,
 101         NVME_CR_VS      = 0x08,
 102         NVME_CR_INTMS   = 0x0c,
 103         NVME_CR_INTMC   = 0x10,
 104         NVME_CR_CC      = 0x14,
 105         NVME_CR_CSTS    = 0x1c,
 106         NVME_CR_NSSR    = 0x20,
 107         NVME_CR_AQA     = 0x24,
 108         NVME_CR_ASQ_LOW = 0x28,
 109         NVME_CR_ASQ_HI  = 0x2c,
 110         NVME_CR_ACQ_LOW = 0x30,
 111         NVME_CR_ACQ_HI  = 0x34,
 112 };
 113
 114 enum nvme_cmd_cdw11 {
 115         NVME_CMD_CDW11_PC  = 0x0001,
 116         NVME_CMD_CDW11_IEN = 0x0002,
 117         NVME_CMD_CDW11_IV  = 0xFFFF0000,
 118 };
 119
 120 #define NVME_CMD_GET_OPC(opc) \
 121         ((opc) >> NVME_CMD_OPC_SHIFT & NVME_CMD_OPC_MASK)
 122
 123 #define NVME_CQ_INTEN   0x01
 124 #define NVME_CQ_INTCOAL 0x02
 125
 126 struct nvme_completion_queue {
 127         struct nvme_completion *qbase;
 128         uint32_t        size;
 129         uint16_t        tail; /* nvme progress */
 130         uint16_t        head; /* guest progress */
 131         uint16_t        intr_vec;
 132         uint32_t        intr_en;
 133         pthread_mutex_t mtx;
 134 };
 135
 136 struct nvme_submission_queue {
 137         struct nvme_command *qbase;
 138         uint32_t        size;
 139         uint16_t        head; /* nvme progress */
 140         uint16_t        tail; /* guest progress */
 141         uint16_t        cqid; /* completion queue id */
 142         int             busy; /* queue is being processed */
 143         int             qpriority;
 144 };
 145
 146 enum nvme_storage_type {
 147         NVME_STOR_BLOCKIF = 0,
 148         NVME_STOR_RAM = 1,
 149 };
 150
 151 struct pci_nvme_blockstore {
 152         enum nvme_storage_type type;
 153         void            *ctx;
 154         uint64_t        size;
 155         uint32_t        sectsz;
 156         uint32_t        sectsz_bits;
 157 };
 158
 159 struct pci_nvme_ioreq {
 160         struct pci_nvme_softc *sc;
 161         struct pci_nvme_ioreq *next;
 162         struct nvme_submission_queue *nvme_sq;
 163         uint16_t        sqid;
 164
 165         /* command information */
 166         uint16_t        opc;
 167         uint16_t        cid;
 168         uint32_t        nsid;
 169
 170         uint64_t        prev_gpaddr;
 171         size_t          prev_size;
 172
 173         /*
 174          * lock if all iovs consumed (big IO);
 175          * complete transaction before continuing
 176          */
 177         pthread_mutex_t mtx;
 178         pthread_cond_t  cv;
 179
 180         struct blockif_req io_req;
 181
 182         /* pad to fit up to 512 page descriptors from guest IO request */
 183         struct iovec    iovpadding[NVME_MAX_BLOCKIOVS-BLOCKIF_IOV_MAX];
 184 };
 185
 186 struct pci_nvme_softc {
 187         struct pci_devinst *nsc_pi;
 188
 189         pthread_mutex_t mtx;
 190
 191         struct nvme_registers regs;
 192
 193         struct nvme_namespace_data  nsdata;
 194         struct nvme_controller_data ctrldata;
 195
 196         struct pci_nvme_blockstore nvstore;
 197
 198         uint16_t        max_qentries; /* max entries per queue */
 199         uint32_t        max_queues;
 200         uint32_t        num_cqueues;
 201         uint32_t        num_squeues;
 202
 203         struct pci_nvme_ioreq *ioreqs;
 204         struct pci_nvme_ioreq *ioreqs_free; /* free list of ioreqs */
 205         uint32_t        pending_ios;
 206         uint32_t        ioslots;
 207         sem_t           iosemlock;
 208
 209         /* status and guest memory mapped queues */
 210         struct nvme_completion_queue *compl_queues;
 211         struct nvme_submission_queue *submit_queues;
 212
 213         /* controller features */
 214         uint32_t        intr_coales_aggr_time;   /* 0x08: uS to delay intr */
 215         uint32_t        intr_coales_aggr_thresh; /* 0x08: compl-Q entries */
 216         uint32_t        async_ev_config;         /* 0x0B: async event config */
 217 };
 218
 219
 220 static void pci_nvme_io_partial(struct blockif_req *br, int err);
 221
 222 /* Controller Configuration utils */
 223 #define NVME_CC_GET_EN(cc) \
 224         ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK)
 225 #define NVME_CC_GET_CSS(cc) \
 226         ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK)
 227 #define NVME_CC_GET_SHN(cc) \
 228         ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK)
 229 #define NVME_CC_GET_IOSQES(cc) \
 230         ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK)
 231 #define NVME_CC_GET_IOCQES(cc) \
 232         ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK)
 233
 234 #define NVME_CC_WRITE_MASK \
 235         ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \
 236          (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \
 237          (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT))
 238
 239 #define NVME_CC_NEN_WRITE_MASK \
 240         ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \
 241          (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \
 242          (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT))
 243
 244 /* Controller Status utils */
 245 #define NVME_CSTS_GET_RDY(sts) \
 246         ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK)
 247
 248 #define NVME_CSTS_RDY   (1 << NVME_CSTS_REG_RDY_SHIFT)
 249
 250 /* Completion Queue status word utils */
 251 #define NVME_STATUS_P   (1 << NVME_STATUS_P_SHIFT)
 252 #define NVME_STATUS_MASK \
 253         ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\
 254          (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT))
 255
 256 static __inline void
 257 pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code)
 258 {
 259
 260         *status &= ~NVME_STATUS_MASK;
 261         *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT |
 262                 (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT;
 263 }
 264
 265 static __inline void
 266 pci_nvme_status_genc(uint16_t *status, uint16_t code)
 267 {
 268
 269         pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);
 270 }
 271
 272 static __inline void
 273 pci_nvme_toggle_phase(uint16_t *status, int prev)
 274 {
 275
 276         if (prev)
 277                 *status &= ~NVME_STATUS_P;
 278         else
 279                 *status |= NVME_STATUS_P;
 280 }
 281
 282 static void
 283 pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)
 284 {
 285         struct nvme_controller_data *cd = &sc->ctrldata;
 286
 287         cd->vid = 0xFB5D;
 288         cd->ssvid = 0x0000;
 289
 290         cd->mn[0] = 'b';
 291         cd->mn[1] = 'h';
 292         cd->mn[2] = 'y';
 293         cd->mn[3] = 'v';
 294         cd->mn[4] = 'e';
 295         cd->mn[5] = '-';
 296         cd->mn[6] = 'N';
 297         cd->mn[7] = 'V';
 298         cd->mn[8] = 'M';
 299         cd->mn[9] = 'e';
 300
 301         cd->fr[0] = '1';
 302         cd->fr[1] = '.';
 303         cd->fr[2] = '0';
 304
 305         /* Num of submission commands that we can handle at a time (2^rab) */
 306         cd->rab   = 4;
 307
 308         /* FreeBSD OUI */
 309         cd->ieee[0] = 0x58;
 310         cd->ieee[1] = 0x9c;
 311         cd->ieee[2] = 0xfc;
 312
 313         cd->mic = 0;
 314
 315         cd->mdts = 9;   /* max data transfer size (2^mdts * CAP.MPSMIN) */
 316
 317         cd->ver = 0x00010300;
 318
 319         cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT;
 320         cd->acl = 2;
 321         cd->aerl = 4;
 322
 323         cd->lpa = 0;    /* TODO: support some simple things like SMART */
 324         cd->elpe = 0;   /* max error log page entries */
 325         cd->npss = 1;   /* number of power states support */
 326
 327         /* Warning Composite Temperature Threshold */
 328         cd->wctemp = 0x0157;
 329
 330         cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) |
 331             (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT);
 332         cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) |
 333             (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT);
 334         cd->nn = 1;     /* number of namespaces */
 335
 336         cd->fna = 0x03;
 337
 338         cd->power_state[0].mp = 10;
 339 }
 340
 341 static void
 342 pci_nvme_init_nsdata(struct pci_nvme_softc *sc)
 343 {
 344         struct nvme_namespace_data *nd;
 345
 346         nd = &sc->nsdata;
 347
 348         nd->nsze = sc->nvstore.size / sc->nvstore.sectsz;
 349         nd->ncap = nd->nsze;
 350         nd->nuse = nd->nsze;
 351
 352         /* Get LBA and backstore information from backing store */
 353         nd->nlbaf = 1;
 354         /* LBA data-sz = 2^lbads */
 355         nd->lbaf[0] = sc->nvstore.sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT;
 356
 357         nd->flbas = 0;
 358 }
 359
 360 static void
 361 pci_nvme_reset(struct pci_nvme_softc *sc)
 362 {
 363         DPRINTF(("%s\r\n", __func__));
 364
 365         sc->regs.cap_lo = (sc->max_qentries & NVME_CAP_LO_REG_MQES_MASK) |
 366             (1 << NVME_CAP_LO_REG_CQR_SHIFT) |
 367             (60 << NVME_CAP_LO_REG_TO_SHIFT);
 368
 369         sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;
 370
 371         sc->regs.vs = 0x00010300;       /* NVMe v1.3 */
 372
 373         sc->regs.cc = 0;
 374         sc->regs.csts = 0;
 375
 376         if (sc->submit_queues != NULL) {
 377                 pthread_mutex_lock(&sc->mtx);
 378                 sc->num_cqueues = sc->num_squeues = sc->max_queues;
 379
 380                 for (int i = 0; i <= sc->max_queues; i++) {
 381                         /*
 382                          * The Admin Submission Queue is at index 0.
 383                          * It must not be changed at reset otherwise the
 384                          * emulation will be out of sync with the guest.
 385                          */
 386                         if (i != 0) {
 387                                 sc->submit_queues[i].qbase = NULL;
 388                                 sc->submit_queues[i].size = 0;
 389                                 sc->submit_queues[i].cqid = 0;
 390
 391                                 sc->compl_queues[i].qbase = NULL;
 392                                 sc->compl_queues[i].size = 0;
 393                         }
 394                         sc->submit_queues[i].tail = 0;
 395                         sc->submit_queues[i].head = 0;
 396                         sc->submit_queues[i].busy = 0;
 397
 398                         sc->compl_queues[i].tail = 0;
 399                         sc->compl_queues[i].head = 0;
 400                 }
 401
 402                 pthread_mutex_unlock(&sc->mtx);
 403         } else
 404                 sc->submit_queues = calloc(sc->max_queues + 1,
 405                                         sizeof(struct nvme_submission_queue));
 406
 407         if (sc->compl_queues == NULL) {
 408                 sc->compl_queues = calloc(sc->max_queues + 1,
 409                                         sizeof(struct nvme_completion_queue));
 410
 411                 for (int i = 0; i <= sc->num_cqueues; i++)
 412                         pthread_mutex_init(&sc->compl_queues[i].mtx, NULL);
 413         }
 414 }
 415
 416 static void
 417 pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc)
 418 {
 419         uint16_t acqs, asqs;
 420
 421         DPRINTF(("%s\r\n", __func__));
 422
 423         asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1;
 424         sc->submit_queues[0].size = asqs;
 425         sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq,
 426                     sizeof(struct nvme_command) * asqs);
 427
 428         DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p\r\n",
 429                 __func__, sc->regs.asq, sc->submit_queues[0].qbase));
 430
 431         acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
 432             NVME_AQA_REG_ACQS_MASK) + 1;
 433         sc->compl_queues[0].size = acqs;
 434         sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,
 435                  sizeof(struct nvme_completion) * acqs);
 436         DPRINTF(("%s mapping Admin-CQ guest 0x%lx, host: %p\r\n",
 437                 __func__, sc->regs.acq, sc->compl_queues[0].qbase));
 438 }
 439
 440 static int
 441 nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
 442         struct nvme_completion* compl)
 443 {
 444         uint16_t qid = command->cdw10 & 0xffff;
 445
 446         DPRINTF(("%s DELETE_IO_SQ %u\r\n", __func__, qid));
 447         if (qid == 0 || qid > sc->num_cqueues) {
 448                 WPRINTF(("%s NOT PERMITTED queue id %u / num_squeues %u\r\n",
 449                         __func__, qid, sc->num_squeues));
 450                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 451                     NVME_SC_INVALID_QUEUE_IDENTIFIER);
 452                 return (1);
 453         }
 454
 455         sc->submit_queues[qid].qbase = NULL;
 456         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 457         return (1);
 458 }
 459
 460 static int
 461 nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
 462         struct nvme_completion* compl)
 463 {
 464         if (command->cdw11 & NVME_CMD_CDW11_PC) {
 465                 uint16_t qid = command->cdw10 & 0xffff;
 466                 struct nvme_submission_queue *nsq;
 467
 468                 if (qid > sc->num_squeues) {
 469                         WPRINTF(("%s queue index %u > num_squeues %u\r\n",
 470                                 __func__, qid, sc->num_squeues));
 471                         pci_nvme_status_tc(&compl->status,
 472                             NVME_SCT_COMMAND_SPECIFIC,
 473                             NVME_SC_INVALID_QUEUE_IDENTIFIER);
 474                         return (1);
 475                 }
 476
 477                 nsq = &sc->submit_queues[qid];
 478                 nsq->size = ((command->cdw10 >> 16) & 0xffff) + 1;
 479
 480                 nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 481                               sizeof(struct nvme_command) * (size_t)nsq->size);
 482                 nsq->cqid = (command->cdw11 >> 16) & 0xffff;
 483                 nsq->qpriority = (command->cdw11 >> 1) & 0x03;
 484
 485                 DPRINTF(("%s sq %u size %u gaddr %p cqid %u\r\n", __func__,
 486                         qid, nsq->size, nsq->qbase, nsq->cqid));
 487
 488                 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 489
 490                 DPRINTF(("%s completed creating IOSQ qid %u\r\n",
 491                          __func__, qid));
 492         } else {
 493                 /*
 494                  * Guest sent non-cont submission queue request.
 495                  * This setting is unsupported by this emulation.
 496                  */
 497                 WPRINTF(("%s unsupported non-contig (list-based) "
 498                          "create i/o submission queue\r\n", __func__));
 499
 500                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 501         }
 502         return (1);
 503 }
 504
 505 static int
 506 nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
 507         struct nvme_completion* compl)
 508 {
 509         uint16_t qid = command->cdw10 & 0xffff;
 510
 511         DPRINTF(("%s DELETE_IO_CQ %u\r\n", __func__, qid));
 512         if (qid == 0 || qid > sc->num_cqueues) {
 513                 WPRINTF(("%s queue index %u / num_cqueues %u\r\n",
 514                         __func__, qid, sc->num_cqueues));
 515                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 516                     NVME_SC_INVALID_QUEUE_IDENTIFIER);
 517                 return (1);
 518         }
 519
 520         sc->compl_queues[qid].qbase = NULL;
 521         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 522         return (1);
 523 }
 524
 525 static int
 526 nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
 527         struct nvme_completion* compl)
 528 {
 529         if (command->cdw11 & NVME_CMD_CDW11_PC) {
 530                 uint16_t qid = command->cdw10 & 0xffff;
 531                 struct nvme_completion_queue *ncq;
 532
 533                 if (qid > sc->num_cqueues) {
 534                         WPRINTF(("%s queue index %u > num_cqueues %u\r\n",
 535                                 __func__, qid, sc->num_cqueues));
 536                         pci_nvme_status_tc(&compl->status,
 537                             NVME_SCT_COMMAND_SPECIFIC,
 538                             NVME_SC_INVALID_QUEUE_IDENTIFIER);
 539                         return (1);
 540                 }
 541
 542                 ncq = &sc->compl_queues[qid];
 543                 ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1;
 544                 ncq->intr_vec = (command->cdw11 >> 16) & 0xffff;
 545                 ncq->size = ((command->cdw10 >> 16) & 0xffff) + 1;
 546
 547                 ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
 548                              command->prp1,
 549                              sizeof(struct nvme_command) * (size_t)ncq->size);
 550
 551                 pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 552         } else {
 553                 /*
 554                  * Non-contig completion queue unsupported.
 555                  */
 556                 WPRINTF(("%s unsupported non-contig (list-based) "
 557                          "create i/o completion queue\r\n",
 558                          __func__));
 559
 560                 /* 0x12 = Invalid Use of Controller Memory Buffer */
 561                 pci_nvme_status_genc(&compl->status, 0x12);
 562         }
 563
 564         return (1);
 565 }
 566
 567 static int
 568 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command,
 569         struct nvme_completion* compl)
 570 {
 571         uint32_t logsize = (1 + ((command->cdw10 >> 16) & 0xFFF)) * 2;
 572         uint8_t logpage = command->cdw10 & 0xFF;
 573         void *data;
 574
 575         DPRINTF(("%s log page %u len %u\r\n", __func__, logpage, logsize));
 576
 577         if (logpage >= 1 && logpage <= 3)
 578                 data = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 579                                   PAGE_SIZE);
 580
 581         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 582
 583         switch (logpage) {
 584         case 0x01: /* Error information */
 585                 memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
 586                 break;
 587         case 0x02: /* SMART/Health information */
 588                 /* TODO: present some smart info */
 589                 memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
 590                 break;
 591         case 0x03: /* Firmware slot information */
 592                 memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
 593                 break;
 594         default:
 595                 WPRINTF(("%s get log page %x command not supported\r\n",
 596                         __func__, logpage));
 597
 598                 pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 599                     NVME_SC_INVALID_LOG_PAGE);
 600         }
 601
 602         return (1);
 603 }
 604
 605 static int
 606 nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command,
 607         struct nvme_completion* compl)
 608 {
 609         void *dest;
 610
 611         DPRINTF(("%s identify 0x%x nsid 0x%x\r\n", __func__,
 612                 command->cdw10 & 0xFF, command->nsid));
 613
 614         switch (command->cdw10 & 0xFF) {
 615         case 0x00: /* return Identify Namespace data structure */
 616                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 617                                   sizeof(sc->nsdata));
 618                 memcpy(dest, &sc->nsdata, sizeof(sc->nsdata));
 619                 break;
 620         case 0x01: /* return Identify Controller data structure */
 621                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 622                                   sizeof(sc->ctrldata));
 623                 memcpy(dest, &sc->ctrldata, sizeof(sc->ctrldata));
 624                 break;
 625         case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */
 626                 dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
 627                                   sizeof(uint32_t) * 1024);
 628                 ((uint32_t *)dest)[0] = 1;
 629                 ((uint32_t *)dest)[1] = 0;
 630                 break;
 631         case 0x11:
 632                 pci_nvme_status_genc(&compl->status,
 633                     NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
 634                 return (1);
 635         case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */
 636         case 0x10:
 637         case 0x12:
 638         case 0x13:
 639         case 0x14:
 640         case 0x15:
 641         default:
 642                 DPRINTF(("%s unsupported identify command requested 0x%x\r\n",
 643                          __func__, command->cdw10 & 0xFF));
 644                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 645                 return (1);
 646         }
 647
 648         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 649         return (1);
 650 }
 651
 652 static int
 653 nvme_opc_set_features(struct pci_nvme_softc* sc, struct nvme_command* command,
 654         struct nvme_completion* compl)
 655 {
 656         int feature = command->cdw10 & 0x0F;
 657         uint32_t iv;
 658
 659         DPRINTF(("%s feature 0x%x\r\n", __func__, feature));
 660         compl->cdw0 = 0;
 661
 662         switch (feature) {
 663         case NVME_FEAT_ARBITRATION:
 664                 DPRINTF(("  arbitration 0x%x\r\n", command->cdw11));
 665                 break;
 666         case NVME_FEAT_POWER_MANAGEMENT:
 667                 DPRINTF(("  power management 0x%x\r\n", command->cdw11));
 668                 break;
 669         case NVME_FEAT_LBA_RANGE_TYPE:
 670                 DPRINTF(("  lba range 0x%x\r\n", command->cdw11));
 671                 break;
 672         case NVME_FEAT_TEMPERATURE_THRESHOLD:
 673                 DPRINTF(("  temperature threshold 0x%x\r\n", command->cdw11));
 674                 break;
 675         case NVME_FEAT_ERROR_RECOVERY:
 676                 DPRINTF(("  error recovery 0x%x\r\n", command->cdw11));
 677                 break;
 678         case NVME_FEAT_VOLATILE_WRITE_CACHE:
 679                 DPRINTF(("  volatile write cache 0x%x\r\n", command->cdw11));
 680                 break;
 681         case NVME_FEAT_NUMBER_OF_QUEUES:
 682                 sc->num_squeues = command->cdw11 & 0xFFFF;
 683                 sc->num_cqueues = (command->cdw11 >> 16) & 0xFFFF;
 684                 DPRINTF(("  number of queues (submit %u, completion %u)\r\n",
 685                         sc->num_squeues, sc->num_cqueues));
 686
 687                 if (sc->num_squeues == 0 || sc->num_squeues > sc->max_queues)
 688                         sc->num_squeues = sc->max_queues;
 689                 if (sc->num_cqueues == 0 || sc->num_cqueues > sc->max_queues)
 690                         sc->num_cqueues = sc->max_queues;
 691
 692                 compl->cdw0 = (sc->num_squeues & 0xFFFF) |
 693                               ((sc->num_cqueues & 0xFFFF) << 16);
 694
 695                 break;
 696         case NVME_FEAT_INTERRUPT_COALESCING:
 697                 DPRINTF(("  interrupt coalescing 0x%x\r\n", command->cdw11));
 698
 699                 /* in uS */
 700                 sc->intr_coales_aggr_time = ((command->cdw11 >> 8) & 0xFF)*100;
 701
 702                 sc->intr_coales_aggr_thresh = command->cdw11 & 0xFF;
 703                 break;
 704         case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
 705                 iv = command->cdw11 & 0xFFFF;
 706
 707                 DPRINTF(("  interrupt vector configuration 0x%x\r\n",
 708                         command->cdw11));
 709
 710                 for (uint32_t i = 0; i <= sc->num_cqueues; i++) {
 711                         if (sc->compl_queues[i].intr_vec == iv) {
 712                                 if (command->cdw11 & (1 << 16))
 713                                         sc->compl_queues[i].intr_en |=
 714                                                               NVME_CQ_INTCOAL;
 715                                 else
 716                                         sc->compl_queues[i].intr_en &=
 717                                                              ~NVME_CQ_INTCOAL;
 718                         }
 719                 }
 720                 break;
 721         case NVME_FEAT_WRITE_ATOMICITY:
 722                 DPRINTF(("  write atomicity 0x%x\r\n", command->cdw11));
 723                 break;
 724         case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
 725                 DPRINTF(("  async event configuration 0x%x\r\n",
 726                         command->cdw11));
 727                 sc->async_ev_config = command->cdw11;
 728                 break;
 729         case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
 730                 DPRINTF(("  software progress marker 0x%x\r\n",
 731                         command->cdw11));
 732                 break;
 733         case 0x0C:
 734                 DPRINTF(("  autonomous power state transition 0x%x\r\n",
 735                         command->cdw11));
 736                 break;
 737         default:
 738                 WPRINTF(("%s invalid feature\r\n", __func__));
 739                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 740                 return (1);
 741         }
 742
 743         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 744         return (1);
 745 }
 746
 747 static int
 748 nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command,
 749         struct nvme_completion* compl)
 750 {
 751         int feature = command->cdw10 & 0x0F;
 752
 753         DPRINTF(("%s feature 0x%x\r\n", __func__, feature));
 754
 755         compl->cdw0 = 0;
 756
 757         switch (feature) {
 758         case NVME_FEAT_ARBITRATION:
 759                 DPRINTF(("  arbitration\r\n"));
 760                 break;
 761         case NVME_FEAT_POWER_MANAGEMENT:
 762                 DPRINTF(("  power management\r\n"));
 763                 break;
 764         case NVME_FEAT_LBA_RANGE_TYPE:
 765                 DPRINTF(("  lba range\r\n"));
 766                 break;
 767         case NVME_FEAT_TEMPERATURE_THRESHOLD:
 768                 DPRINTF(("  temperature threshold\r\n"));
 769                 switch ((command->cdw11 >> 20) & 0x3) {
 770                 case 0:
 771                         /* Over temp threshold */
 772                         compl->cdw0 = 0xFFFF;
 773                         break;
 774                 case 1:
 775                         /* Under temp threshold */
 776                         compl->cdw0 = 0;
 777                         break;
 778                 default:
 779                         WPRINTF(("  invalid threshold type select\r\n"));
 780                         pci_nvme_status_genc(&compl->status,
 781                             NVME_SC_INVALID_FIELD);
 782                         return (1);
 783                 }
 784                 break;
 785         case NVME_FEAT_ERROR_RECOVERY:
 786                 DPRINTF(("  error recovery\r\n"));
 787                 break;
 788         case NVME_FEAT_VOLATILE_WRITE_CACHE:
 789                 DPRINTF(("  volatile write cache\r\n"));
 790                 break;
 791         case NVME_FEAT_NUMBER_OF_QUEUES:
 792                 compl->cdw0 = 0;
 793                 if (sc->num_squeues == 0)
 794                         compl->cdw0 |= sc->max_queues & 0xFFFF;
 795                 else
 796                         compl->cdw0 |= sc->num_squeues & 0xFFFF;
 797
 798                 if (sc->num_cqueues == 0)
 799                         compl->cdw0 |= (sc->max_queues & 0xFFFF) << 16;
 800                 else
 801                         compl->cdw0 |= (sc->num_cqueues & 0xFFFF) << 16;
 802
 803                 DPRINTF(("  number of queues (submit %u, completion %u)\r\n",
 804                         compl->cdw0 & 0xFFFF,
 805                         (compl->cdw0 >> 16) & 0xFFFF));
 806
 807                 break;
 808         case NVME_FEAT_INTERRUPT_COALESCING:
 809                 DPRINTF(("  interrupt coalescing\r\n"));
 810                 break;
 811         case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
 812                 DPRINTF(("  interrupt vector configuration\r\n"));
 813                 break;
 814         case NVME_FEAT_WRITE_ATOMICITY:
 815                 DPRINTF(("  write atomicity\r\n"));
 816                 break;
 817         case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
 818                 DPRINTF(("  async event configuration\r\n"));
 819                 sc->async_ev_config = command->cdw11;
 820                 break;
 821         case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
 822                 DPRINTF(("  software progress marker\r\n"));
 823                 break;
 824         case 0x0C:
 825                 DPRINTF(("  autonomous power state transition\r\n"));
 826                 break;
 827         default:
 828                 WPRINTF(("%s invalid feature 0x%x\r\n", __func__, feature));
 829                 pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
 830                 return (1);
 831         }
 832
 833         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 834         return (1);
 835 }
 836
 837 static int
 838 nvme_opc_abort(struct pci_nvme_softc* sc, struct nvme_command* command,
 839         struct nvme_completion* compl)
 840 {
 841         DPRINTF(("%s submission queue %u, command ID 0x%x\r\n", __func__,
 842                 command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF));
 843
 844         /* TODO: search for the command ID and abort it */
 845
 846         compl->cdw0 = 1;
 847         pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
 848         return (1);
 849 }
 850
 851 static int
 852 nvme_opc_async_event_req(struct pci_nvme_softc* sc,
 853         struct nvme_command* command, struct nvme_completion* compl)
 854 {
 855         DPRINTF(("%s async event request 0x%x\r\n", __func__, command->cdw11));
 856
 857         /*
 858          * TODO: raise events when they happen based on the Set Features cmd.
 859          * These events happen async, so only set completion successful if
 860          * there is an event reflective of the request to get event.
 861          */
 862         pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
 863             NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
 864         return (0);
 865 }
 866
 867 static void
 868 pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
 869 {
 870         struct nvme_completion compl;
 871         struct nvme_command *cmd;
 872         struct nvme_submission_queue *sq;
 873         struct nvme_completion_queue *cq;
 874         int do_intr = 0;
 875         uint16_t sqhead;
 876
 877         DPRINTF(("%s index %u\r\n", __func__, (uint32_t)value));
 878
 879         sq = &sc->submit_queues[0];
 880
 881         sqhead = atomic_load_acq_short(&sq->head);
 882
 883         if (atomic_testandset_int(&sq->busy, 1)) {
 884                 DPRINTF(("%s SQ busy, head %u, tail %u\r\n",
 885                         __func__, sqhead, sq->tail));
 886                 return;
 887         }
 888
 889         DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail));
 890
 891         while (sqhead != atomic_load_acq_short(&sq->tail)) {
 892                 cmd = &(sq->qbase)[sqhead];
 893                 compl.status = 0;
 894
 895                 switch (NVME_CMD_GET_OPC(cmd->opc_fuse)) {
 896                 case NVME_OPC_DELETE_IO_SQ:
 897                         DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__));
 898                         do_intr |= nvme_opc_delete_io_sq(sc, cmd, &compl);
 899                         break;
 900                 case NVME_OPC_CREATE_IO_SQ:
 901                         DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__));
 902                         do_intr |= nvme_opc_create_io_sq(sc, cmd, &compl);
 903                         break;
 904                 case NVME_OPC_DELETE_IO_CQ:
 905                         DPRINTF(("%s command DELETE_IO_CQ\r\n", __func__));
 906                         do_intr |= nvme_opc_delete_io_cq(sc, cmd, &compl);
 907                         break;
 908                 case NVME_OPC_CREATE_IO_CQ:
 909                         DPRINTF(("%s command CREATE_IO_CQ\r\n", __func__));
 910                         do_intr |= nvme_opc_create_io_cq(sc, cmd, &compl);
 911                         break;
 912                 case NVME_OPC_GET_LOG_PAGE:
 913                         DPRINTF(("%s command GET_LOG_PAGE\r\n", __func__));
 914                         do_intr |= nvme_opc_get_log_page(sc, cmd, &compl);
 915                         break;
 916                 case NVME_OPC_IDENTIFY:
 917                         DPRINTF(("%s command IDENTIFY\r\n", __func__));
 918                         do_intr |= nvme_opc_identify(sc, cmd, &compl);
 919                         break;
 920                 case NVME_OPC_ABORT:
 921                         DPRINTF(("%s command ABORT\r\n", __func__));
 922                         do_intr |= nvme_opc_abort(sc, cmd, &compl);
 923                         break;
 924                 case NVME_OPC_SET_FEATURES:
 925                         DPRINTF(("%s command SET_FEATURES\r\n", __func__));
 926                         do_intr |= nvme_opc_set_features(sc, cmd, &compl);
 927                         break;
 928                 case NVME_OPC_GET_FEATURES:
 929                         DPRINTF(("%s command GET_FEATURES\r\n", __func__));
 930                         do_intr |= nvme_opc_get_features(sc, cmd, &compl);
 931                         break;
 932                 case NVME_OPC_ASYNC_EVENT_REQUEST:
 933                         DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__));
 934                         /* XXX dont care, unhandled for now
 935                         do_intr |= nvme_opc_async_event_req(sc, cmd, &compl);
 936                         */
 937                         break;
 938                 default:
 939                         WPRINTF(("0x%x command is not implemented\r\n",
 940                             NVME_CMD_GET_OPC(cmd->opc_fuse)));
 941                 }
 942
 943                 /* for now skip async event generation */
 944                 if (NVME_CMD_GET_OPC(cmd->opc_fuse) !=
 945                     NVME_OPC_ASYNC_EVENT_REQUEST) {
 946                         struct nvme_completion *cp;
 947                         int phase;
 948
 949                         cq = &sc->compl_queues[0];
 950
 951                         cp = &(cq->qbase)[cq->tail];
 952                         cp->sqid = 0;
 953                         cp->sqhd = sqhead;
 954                         cp->cid = cmd->cid;
 955
 956                         phase = NVME_STATUS_GET_P(cp->status);
 957                         cp->status = compl.status;
 958                         pci_nvme_toggle_phase(&cp->status, phase);
 959
 960                         cq->tail = (cq->tail + 1) % cq->size;
 961                 }
 962                 sqhead = (sqhead + 1) % sq->size;
 963         }
 964
 965         DPRINTF(("setting sqhead %u\r\n", sqhead));
 966         atomic_store_short(&sq->head, sqhead);
 967         atomic_store_int(&sq->busy, 0);
 968
 969         if (do_intr)
 970                 pci_generate_msix(sc->nsc_pi, 0);
 971
 972 }
 973
 974 static int
 975 pci_nvme_append_iov_req(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req,
 976         uint64_t gpaddr, size_t size, int do_write, uint64_t lba)
 977 {
 978         int iovidx;
 979
 980         if (req != NULL) {
 981                 /* concatenate contig block-iovs to minimize number of iovs */
 982                 if ((req->prev_gpaddr + req->prev_size) == gpaddr) {
 983                         iovidx = req->io_req.br_iovcnt - 1;
 984
 985                         req->io_req.br_iov[iovidx].iov_base =
 986                             paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
 987                                              req->prev_gpaddr, size);
 988
 989                         req->prev_size += size;
 990                         req->io_req.br_resid += size;
 991
 992                         req->io_req.br_iov[iovidx].iov_len = req->prev_size;
 993                 } else {
 994                         pthread_mutex_lock(&req->mtx);
 995
 996                         iovidx = req->io_req.br_iovcnt;
 997                         if (iovidx == NVME_MAX_BLOCKIOVS) {
 998                                 int err = 0;
 999
1000                                 DPRINTF(("large I/O, doing partial req\r\n"));
1001
1002                                 iovidx = 0;
1003                                 req->io_req.br_iovcnt = 0;
1004
1005                                 req->io_req.br_callback = pci_nvme_io_partial;
1006
1007                                 if (!do_write)
1008                                         err = blockif_read(sc->nvstore.ctx,
1009                                                            &req->io_req);
1010                                 else
1011                                         err = blockif_write(sc->nvstore.ctx,
1012                                                             &req->io_req);
1013
1014                                 /* wait until req completes before cont */
1015                                 if (err == 0)
1016                                         pthread_cond_wait(&req->cv, &req->mtx);
1017                         }
1018                         if (iovidx == 0) {
1019                                 req->io_req.br_offset = lba;
1020                                 req->io_req.br_resid = 0;
1021                                 req->io_req.br_param = req;
1022                         }
1023
1024                         req->io_req.br_iov[iovidx].iov_base =
1025                             paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
1026                                              gpaddr, size);
1027
1028                         req->io_req.br_iov[iovidx].iov_len = size;
1029
1030                         req->prev_gpaddr = gpaddr;
1031                         req->prev_size = size;
1032                         req->io_req.br_resid += size;
1033
1034                         req->io_req.br_iovcnt++;
1035
1036                         pthread_mutex_unlock(&req->mtx);
1037                 }
1038         } else {
1039                 /* RAM buffer: read/write directly */
1040                 void *p = sc->nvstore.ctx;
1041                 void *gptr;
1042
1043                 if ((lba + size) > sc->nvstore.size) {
1044                         WPRINTF(("%s write would overflow RAM\r\n", __func__));
1045                         return (-1);
1046                 }
1047
1048                 p = (void *)((uintptr_t)p + (uintptr_t)lba);
1049                 gptr = paddr_guest2host(sc->nsc_pi->pi_vmctx, gpaddr, size);
1050                 if (do_write)
1051                         memcpy(p, gptr, size);
1052                 else
1053                         memcpy(gptr, p, size);
1054         }
1055         return (0);
1056 }
1057
1058 static void
1059 pci_nvme_set_completion(struct pci_nvme_softc *sc,
1060         struct nvme_submission_queue *sq, int sqid, uint16_t cid,
1061         uint32_t cdw0, uint16_t status, int ignore_busy)
1062 {
1063         struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
1064         struct nvme_completion *compl;
1065         int do_intr = 0;
1066         int phase;
1067
1068         DPRINTF(("%s sqid %d cqid %u cid %u status: 0x%x 0x%x\r\n",
1069                  __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
1070                  NVME_STATUS_GET_SC(status)));
1071
1072         pthread_mutex_lock(&cq->mtx);
1073
1074         assert(cq->qbase != NULL);
1075
1076         compl = &cq->qbase[cq->tail];
1077
1078         compl->sqhd = atomic_load_acq_short(&sq->head);
1079         compl->sqid = sqid;
1080         compl->cid = cid;
1081
1082         // toggle phase
1083         phase = NVME_STATUS_GET_P(compl->status);
1084         compl->status = status;
1085         pci_nvme_toggle_phase(&compl->status, phase);
1086
1087         cq->tail = (cq->tail + 1) % cq->size;
1088
1089         if (cq->intr_en & NVME_CQ_INTEN)
1090                 do_intr = 1;
1091
1092         pthread_mutex_unlock(&cq->mtx);
1093
1094         if (ignore_busy || !atomic_load_acq_int(&sq->busy))
1095                 if (do_intr)
1096                         pci_generate_msix(sc->nsc_pi, cq->intr_vec);
1097 }
1098
1099 static void
1100 pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req)
1101 {
1102         req->sc = NULL;
1103         req->nvme_sq = NULL;
1104         req->sqid = 0;
1105
1106         pthread_mutex_lock(&sc->mtx);
1107
1108         req->next = sc->ioreqs_free;
1109         sc->ioreqs_free = req;
1110         sc->pending_ios--;
1111
1112         /* when no more IO pending, can set to ready if device reset/enabled */
1113         if (sc->pending_ios == 0 &&
1114             NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts)))
1115                 sc->regs.csts |= NVME_CSTS_RDY;
1116
1117         pthread_mutex_unlock(&sc->mtx);
1118
1119         sem_post(&sc->iosemlock);
1120 }
1121
1122 static struct pci_nvme_ioreq *
1123 pci_nvme_get_ioreq(struct pci_nvme_softc *sc)
1124 {
1125         struct pci_nvme_ioreq *req = NULL;;
1126
1127         sem_wait(&sc->iosemlock);
1128         pthread_mutex_lock(&sc->mtx);
1129
1130         req = sc->ioreqs_free;
1131         assert(req != NULL);
1132
1133         sc->ioreqs_free = req->next;
1134
1135         req->next = NULL;
1136         req->sc = sc;
1137
1138         sc->pending_ios++;
1139
1140         pthread_mutex_unlock(&sc->mtx);
1141
1142         req->io_req.br_iovcnt = 0;
1143         req->io_req.br_offset = 0;
1144         req->io_req.br_resid = 0;
1145         req->io_req.br_param = req;
1146         req->prev_gpaddr = 0;
1147         req->prev_size = 0;
1148
1149         return req;
1150 }
1151
1152 static void
1153 pci_nvme_io_done(struct blockif_req *br, int err)
1154 {
1155         struct pci_nvme_ioreq *req = br->br_param;
1156         struct nvme_submission_queue *sq = req->nvme_sq;
1157         uint16_t code, status;
1158
1159         DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err)));
1160
1161         /* TODO return correct error */
1162         if (err)
1163                 code = NVME_SC_DATA_TRANSFER_ERROR;
1164         else
1165                 code = NVME_SC_SUCCESS;
1166
1167         code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;
1168         pci_nvme_status_genc(&status, code);
1169
1170         pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status, 0);
1171         pci_nvme_release_ioreq(req->sc, req);
1172 }
1173
1174 static void
1175 pci_nvme_io_partial(struct blockif_req *br, int err)
1176 {
1177         struct pci_nvme_ioreq *req = br->br_param;
1178
1179         DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err)));
1180
1181         pthread_cond_signal(&req->cv);
1182 }
1183
1184
1185 static void
1186 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx)
1187 {
1188         struct nvme_submission_queue *sq;
1189         uint16_t status;
1190         uint16_t sqhead;
1191         int err;
1192
1193         /* handle all submissions up to sq->tail index */
1194         sq = &sc->submit_queues[idx];
1195
1196         if (atomic_testandset_int(&sq->busy, 1)) {
1197                 DPRINTF(("%s sqid %u busy\r\n", __func__, idx));
1198                 return;
1199         }
1200
1201         sqhead = atomic_load_acq_short(&sq->head);
1202
1203         DPRINTF(("nvme_handle_io qid %u head %u tail %u cmdlist %p\r\n",
1204                  idx, sqhead, sq->tail, sq->qbase));
1205
1206         while (sqhead != atomic_load_acq_short(&sq->tail)) {
1207                 struct nvme_command *cmd;
1208                 struct pci_nvme_ioreq *req = NULL;
1209                 uint64_t lba;
1210                 uint64_t nblocks, bytes, size, cpsz;
1211
1212                 /* TODO: support scatter gather list handling */
1213
1214                 cmd = &sq->qbase[sqhead];
1215                 sqhead = (sqhead + 1) % sq->size;
1216
1217                 lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
1218
1219                 if (NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_FLUSH) {
1220                         pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1221                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1222                                                 status, 1);
1223
1224                         continue;
1225                 } else if (NVME_CMD_GET_OPC(cmd->opc_fuse) == 0x08) {
1226                         /* TODO: write zeroes */
1227                         WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n",
1228                                 __func__, lba, cmd->cdw12 & 0xFFFF));
1229                         pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1230                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1231                                                 status, 1);
1232
1233                         continue;
1234                 }
1235
1236                 nblocks = (cmd->cdw12 & 0xFFFF) + 1;
1237
1238                 bytes = nblocks * sc->nvstore.sectsz;
1239
1240                 if (sc->nvstore.type == NVME_STOR_BLOCKIF) {
1241                         req = pci_nvme_get_ioreq(sc);
1242                         req->nvme_sq = sq;
1243                         req->sqid = idx;
1244                 }
1245
1246                 /*
1247                  * If data starts mid-page and flows into the next page, then
1248                  * increase page count
1249                  */
1250
1251                 DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu "
1252                          "(%lu-bytes)\r\n",
1253                          sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size,
1254                          NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE ?
1255                              "WRITE" : "READ",
1256                          lba, nblocks, bytes));
1257
1258                 cmd->prp1 &= ~(0x03UL);
1259                 cmd->prp2 &= ~(0x03UL);
1260
1261                 DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2));
1262
1263                 size = bytes;
1264                 lba *= sc->nvstore.sectsz;
1265
1266                 cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE);
1267
1268                 if (cpsz > bytes)
1269                         cpsz = bytes;
1270
1271                 if (req != NULL) {
1272                         req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) |
1273                                                 cmd->cdw10;
1274                         req->opc = NVME_CMD_GET_OPC(cmd->opc_fuse);
1275                         req->cid = cmd->cid;
1276                         req->nsid = cmd->nsid;
1277                 }
1278
1279                 err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz,
1280                     NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE, lba);
1281                 lba += cpsz;
1282                 size -= cpsz;
1283
1284                 if (size == 0)
1285                         goto iodone;
1286
1287                 if (size <= PAGE_SIZE) {
1288                         /* prp2 is second (and final) page in transfer */
1289
1290                         err = pci_nvme_append_iov_req(sc, req, cmd->prp2,
1291                             size,
1292                             NVME_CMD_GET_OPC(cmd->opc_fuse) == NVME_OPC_WRITE,
1293                             lba);
1294                 } else {
1295                         uint64_t *prp_list;
1296                         int i;
1297
1298                         /* prp2 is pointer to a physical region page list */
1299                         prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx,
1300                                                     cmd->prp2, PAGE_SIZE);
1301
1302                         i = 0;
1303                         while (size != 0) {
1304                                 cpsz = MIN(size, PAGE_SIZE);
1305
1306                                 /*
1307                                  * Move to linked physical region page list
1308                                  * in last item.
1309                                  */
1310                                 if (i == (NVME_PRP2_ITEMS-1) &&
1311                                     size > PAGE_SIZE) {
1312                                         assert((prp_list[i] & (PAGE_SIZE-1)) == 0);
1313                                         prp_list = paddr_guest2host(
1314                                                       sc->nsc_pi->pi_vmctx,
1315                                                       prp_list[i], PAGE_SIZE);
1316                                         i = 0;
1317                                 }
1318                                 if (prp_list[i] == 0) {
1319                                         WPRINTF(("PRP2[%d] = 0 !!!\r\n", i));
1320                                         err = 1;
1321                                         break;
1322                                 }
1323
1324                                 err = pci_nvme_append_iov_req(sc, req,
1325                                     prp_list[i], cpsz,
1326                                     NVME_CMD_GET_OPC(cmd->opc_fuse) ==
1327                                         NVME_OPC_WRITE, lba);
1328                                 if (err)
1329                                         break;
1330
1331                                 lba += cpsz;
1332                                 size -= cpsz;
1333                                 i++;
1334                         }
1335                 }
1336
1337 iodone:
1338                 if (sc->nvstore.type == NVME_STOR_RAM) {
1339                         uint16_t code, status;
1340
1341                         code = err ? NVME_SC_LBA_OUT_OF_RANGE :
1342                             NVME_SC_SUCCESS;
1343                         pci_nvme_status_genc(&status, code);
1344
1345                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1346                                                 status, 1);
1347
1348                         continue;
1349                 }
1350
1351
1352                 if (err)
1353                         goto do_error;
1354
1355                 req->io_req.br_callback = pci_nvme_io_done;
1356
1357                 err = 0;
1358                 switch (NVME_CMD_GET_OPC(cmd->opc_fuse)) {
1359                 case NVME_OPC_READ:
1360                         err = blockif_read(sc->nvstore.ctx, &req->io_req);
1361                         break;
1362                 case NVME_OPC_WRITE:
1363                         err = blockif_write(sc->nvstore.ctx, &req->io_req);
1364                         break;
1365                 default:
1366                         WPRINTF(("%s unhandled io command 0x%x\r\n",
1367                                  __func__, NVME_CMD_GET_OPC(cmd->opc_fuse)));
1368                         err = 1;
1369                 }
1370
1371 do_error:
1372                 if (err) {
1373                         uint16_t status;
1374
1375                         pci_nvme_status_genc(&status,
1376                             NVME_SC_DATA_TRANSFER_ERROR);
1377
1378                         pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1379                                                 status, 1);
1380                         pci_nvme_release_ioreq(sc, req);
1381                 }
1382         }
1383
1384         atomic_store_short(&sq->head, sqhead);
1385         atomic_store_int(&sq->busy, 0);
1386 }
1387
1388 static void
1389 pci_nvme_handle_doorbell(struct vmctx *ctx, struct pci_nvme_softc* sc,
1390         uint64_t idx, int is_sq, uint64_t value)
1391 {
1392         DPRINTF(("nvme doorbell %lu, %s, val 0x%lx\r\n",
1393                 idx, is_sq ? "SQ" : "CQ", value & 0xFFFF));
1394
1395         if (is_sq) {
1396                 atomic_store_short(&sc->submit_queues[idx].tail,
1397                                    (uint16_t)value);
1398
1399                 if (idx == 0) {
1400                         pci_nvme_handle_admin_cmd(sc, value);
1401                 } else {
1402                         /* submission queue; handle new entries in SQ */
1403                         if (idx > sc->num_squeues) {
1404                                 WPRINTF(("%s SQ index %lu overflow from "
1405                                          "guest (max %u)\r\n",
1406                                          __func__, idx, sc->num_squeues));
1407                                 return;
1408                         }
1409                         pci_nvme_handle_io_cmd(sc, (uint16_t)idx);
1410                 }
1411         } else {
1412                 if (idx > sc->num_cqueues) {
1413                         WPRINTF(("%s queue index %lu overflow from "
1414                                  "guest (max %u)\r\n",
1415                                  __func__, idx, sc->num_cqueues));
1416                         return;
1417                 }
1418
1419                 sc->compl_queues[idx].head = (uint16_t)value;
1420         }
1421 }
1422
1423 static void
1424 pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)
1425 {
1426         const char *s = iswrite ? "WRITE" : "READ";
1427
1428         switch (offset) {
1429         case NVME_CR_CAP_LOW:
1430                 DPRINTF(("%s %s NVME_CR_CAP_LOW\r\n", func, s));
1431                 break;
1432         case NVME_CR_CAP_HI:
1433                 DPRINTF(("%s %s NVME_CR_CAP_HI\r\n", func, s));
1434                 break;
1435         case NVME_CR_VS:
1436                 DPRINTF(("%s %s NVME_CR_VS\r\n", func, s));
1437                 break;
1438         case NVME_CR_INTMS:
1439                 DPRINTF(("%s %s NVME_CR_INTMS\r\n", func, s));
1440                 break;
1441         case NVME_CR_INTMC:
1442                 DPRINTF(("%s %s NVME_CR_INTMC\r\n", func, s));
1443                 break;
1444         case NVME_CR_CC:
1445                 DPRINTF(("%s %s NVME_CR_CC\r\n", func, s));
1446                 break;
1447         case NVME_CR_CSTS:
1448                 DPRINTF(("%s %s NVME_CR_CSTS\r\n", func, s));
1449                 break;
1450         case NVME_CR_NSSR:
1451                 DPRINTF(("%s %s NVME_CR_NSSR\r\n", func, s));
1452                 break;
1453         case NVME_CR_AQA:
1454                 DPRINTF(("%s %s NVME_CR_AQA\r\n", func, s));
1455                 break;
1456         case NVME_CR_ASQ_LOW:
1457                 DPRINTF(("%s %s NVME_CR_ASQ_LOW\r\n", func, s));
1458                 break;
1459         case NVME_CR_ASQ_HI:
1460                 DPRINTF(("%s %s NVME_CR_ASQ_HI\r\n", func, s));
1461                 break;
1462         case NVME_CR_ACQ_LOW:
1463                 DPRINTF(("%s %s NVME_CR_ACQ_LOW\r\n", func, s));
1464                 break;
1465         case NVME_CR_ACQ_HI:
1466                 DPRINTF(("%s %s NVME_CR_ACQ_HI\r\n", func, s));
1467                 break;
1468         default:
1469                 DPRINTF(("unknown nvme bar-0 offset 0x%lx\r\n", offset));
1470         }
1471
1472 }
1473
1474 static void
1475 pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc,
1476         uint64_t offset, int size, uint64_t value)
1477 {
1478         uint32_t ccreg;
1479
1480         if (offset >= NVME_DOORBELL_OFFSET) {
1481                 uint64_t belloffset = offset - NVME_DOORBELL_OFFSET;
1482                 uint64_t idx = belloffset / 8; /* door bell size = 2*int */
1483                 int is_sq = (belloffset % 8) < 4;
1484
1485                 if (belloffset > ((sc->max_queues+1) * 8 - 4)) {
1486                         WPRINTF(("guest attempted an overflow write offset "
1487                                  "0x%lx, val 0x%lx in %s",
1488                                  offset, value, __func__));
1489                         return;
1490                 }
1491
1492                 pci_nvme_handle_doorbell(ctx, sc, idx, is_sq, value);
1493                 return;
1494         }
1495
1496         DPRINTF(("nvme-write offset 0x%lx, size %d, value 0x%lx\r\n",
1497                 offset, size, value));
1498
1499         if (size != 4) {
1500                 WPRINTF(("guest wrote invalid size %d (offset 0x%lx, "
1501                          "val 0x%lx) to bar0 in %s",
1502                          size, offset, value, __func__));
1503                 /* TODO: shutdown device */
1504                 return;
1505         }
1506
1507         pci_nvme_bar0_reg_dumps(__func__, offset, 1);
1508
1509         pthread_mutex_lock(&sc->mtx);
1510
1511         switch (offset) {
1512         case NVME_CR_CAP_LOW:
1513         case NVME_CR_CAP_HI:
1514                 /* readonly */
1515                 break;
1516         case NVME_CR_VS:
1517                 /* readonly */
1518                 break;
1519         case NVME_CR_INTMS:
1520                 /* MSI-X, so ignore */
1521                 break;
1522         case NVME_CR_INTMC:
1523                 /* MSI-X, so ignore */
1524                 break;
1525         case NVME_CR_CC:
1526                 ccreg = (uint32_t)value;
1527
1528                 DPRINTF(("%s NVME_CR_CC en %x css %x shn %x iosqes %u "
1529                          "iocqes %u\r\n",
1530                         __func__,
1531                          NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg),
1532                          NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg),
1533                          NVME_CC_GET_IOCQES(ccreg)));
1534
1535                 if (NVME_CC_GET_SHN(ccreg)) {
1536                         /* perform shutdown - flush out data to backend */
1537                         sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK <<
1538                             NVME_CSTS_REG_SHST_SHIFT);
1539                         sc->regs.csts |= NVME_SHST_COMPLETE <<
1540                             NVME_CSTS_REG_SHST_SHIFT;
1541                 }
1542                 if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) {
1543                         if (NVME_CC_GET_EN(ccreg) == 0)
1544                                 /* transition 1-> causes controller reset */
1545                                 pci_nvme_reset(sc);
1546                         else
1547                                 pci_nvme_init_controller(ctx, sc);
1548                 }
1549
1550                 /* Insert the iocqes, iosqes and en bits from the write */
1551                 sc->regs.cc &= ~NVME_CC_WRITE_MASK;
1552                 sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK;
1553                 if (NVME_CC_GET_EN(ccreg) == 0) {
1554                         /* Insert the ams, mps and css bit fields */
1555                         sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK;
1556                         sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK;
1557                         sc->regs.csts &= ~NVME_CSTS_RDY;
1558                 } else if (sc->pending_ios == 0) {
1559                         sc->regs.csts |= NVME_CSTS_RDY;
1560                 }
1561                 break;
1562         case NVME_CR_CSTS:
1563                 break;
1564         case NVME_CR_NSSR:
1565                 /* ignore writes; don't support subsystem reset */
1566                 break;
1567         case NVME_CR_AQA:
1568                 sc->regs.aqa = (uint32_t)value;
1569                 break;
1570         case NVME_CR_ASQ_LOW:
1571                 sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) |
1572                                (0xFFFFF000 & value);
1573                 break;
1574         case NVME_CR_ASQ_HI:
1575                 sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) |
1576                                (value << 32);
1577                 break;
1578         case NVME_CR_ACQ_LOW:
1579                 sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) |
1580                                (0xFFFFF000 & value);
1581                 break;
1582         case NVME_CR_ACQ_HI:
1583                 sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) |
1584                                (value << 32);
1585                 break;
1586         default:
1587                 DPRINTF(("%s unknown offset 0x%lx, value 0x%lx size %d\r\n",
1588                          __func__, offset, value, size));
1589         }
1590         pthread_mutex_unlock(&sc->mtx);
1591 }
1592
1593 static void
1594 pci_nvme_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
1595                 int baridx, uint64_t offset, int size, uint64_t value)
1596 {
1597         struct pci_nvme_softc* sc = pi->pi_arg;
1598
1599         if (baridx == pci_msix_table_bar(pi) ||
1600             baridx == pci_msix_pba_bar(pi)) {
1601                 DPRINTF(("nvme-write baridx %d, msix: off 0x%lx, size %d, "
1602                          " value 0x%lx\r\n", baridx, offset, size, value));
1603
1604                 pci_emul_msix_twrite(pi, offset, size, value);
1605                 return;
1606         }
1607
1608         switch (baridx) {
1609         case 0:
1610                 pci_nvme_write_bar_0(ctx, sc, offset, size, value);
1611                 break;
1612
1613         default:
1614                 DPRINTF(("%s unknown baridx %d, val 0x%lx\r\n",
1615                          __func__, baridx, value));
1616         }
1617 }
1618
1619 static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc,
1620         uint64_t offset, int size)
1621 {
1622         uint64_t value;
1623
1624         pci_nvme_bar0_reg_dumps(__func__, offset, 0);
1625
1626         if (offset < NVME_DOORBELL_OFFSET) {
1627                 void *p = &(sc->regs);
1628                 pthread_mutex_lock(&sc->mtx);
1629                 memcpy(&value, (void *)((uintptr_t)p + offset), size);
1630                 pthread_mutex_unlock(&sc->mtx);
1631         } else {
1632                 value = 0;
1633                 WPRINTF(("pci_nvme: read invalid offset %ld\r\n", offset));
1634         }
1635
1636         switch (size) {
1637         case 1:
1638                 value &= 0xFF;
1639                 break;
1640         case 2:
1641                 value &= 0xFFFF;
1642                 break;
1643         case 4:
1644                 value &= 0xFFFFFFFF;
1645                 break;
1646         }
1647
1648         DPRINTF(("   nvme-read offset 0x%lx, size %d -> value 0x%x\r\n",
1649                  offset, size, (uint32_t)value));
1650
1651         return (value);
1652 }
1653
1654
1655
1656 static uint64_t
1657 pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx,
1658     uint64_t offset, int size)
1659 {
1660         struct pci_nvme_softc* sc = pi->pi_arg;
1661
1662         if (baridx == pci_msix_table_bar(pi) ||
1663             baridx == pci_msix_pba_bar(pi)) {
1664                 DPRINTF(("nvme-read bar: %d, msix: regoff 0x%lx, size %d\r\n",
1665                         baridx, offset, size));
1666
1667                 return pci_emul_msix_tread(pi, offset, size);
1668         }
1669
1670         switch (baridx) {
1671         case 0:
1672                 return pci_nvme_read_bar_0(sc, offset, size);
1673
1674         default:
1675                 DPRINTF(("unknown bar %d, 0x%lx\r\n", baridx, offset));
1676         }
1677
1678         return (0);
1679 }
1680
1681
1682 static int
1683 pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts)
1684 {
1685         char bident[sizeof("XX:X:X")];
1686         char    *uopt, *xopts, *config;
1687         uint32_t sectsz;
1688         int optidx;
1689
1690         sc->max_queues = NVME_QUEUES;
1691         sc->max_qentries = NVME_MAX_QENTRIES;
1692         sc->ioslots = NVME_IOSLOTS;
1693         sc->num_squeues = sc->max_queues;
1694         sc->num_cqueues = sc->max_queues;
1695         sectsz = 0;
1696
1697         uopt = strdup(opts);
1698         optidx = 0;
1699         snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
1700                  "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
1701         for (xopts = strtok(uopt, ",");
1702              xopts != NULL;
1703              xopts = strtok(NULL, ",")) {
1704
1705                 if ((config = strchr(xopts, '=')) != NULL)
1706                         *config++ = '\0';
1707
1708                 if (!strcmp("maxq", xopts)) {
1709                         sc->max_queues = atoi(config);
1710                 } else if (!strcmp("qsz", xopts)) {
1711                         sc->max_qentries = atoi(config);
1712                 } else if (!strcmp("ioslots", xopts)) {
1713                         sc->ioslots = atoi(config);
1714                 } else if (!strcmp("sectsz", xopts)) {
1715                         sectsz = atoi(config);
1716                 } else if (!strcmp("ser", xopts)) {
1717                         memset(sc->ctrldata.sn, 0, sizeof(sc->ctrldata.sn));
1718                         strncpy(sc->ctrldata.sn, config,
1719                                 sizeof(sc->ctrldata.sn));
1720                 } else if (!strcmp("ram", xopts)) {
1721                         uint64_t sz = strtoull(&xopts[4], NULL, 10);
1722
1723                         sc->nvstore.type = NVME_STOR_RAM;
1724                         sc->nvstore.size = sz * 1024 * 1024;
1725                         sc->nvstore.ctx = calloc(1, sc->nvstore.size);
1726                         sc->nvstore.sectsz = 4096;
1727                         sc->nvstore.sectsz_bits = 12;
1728                         if (sc->nvstore.ctx == NULL) {
1729                                 perror("Unable to allocate RAM");
1730                                 return (-1);
1731                         }
1732                 } else if (optidx == 0) {
1733                         snprintf(bident, sizeof(bident), "%d:%d",
1734                                  sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
1735                         sc->nvstore.ctx = blockif_open(xopts, bident);
1736                         if (sc->nvstore.ctx == NULL) {
1737                                 perror("Could not open backing file");
1738                                 return (-1);
1739                         }
1740                         sc->nvstore.type = NVME_STOR_BLOCKIF;
1741                         sc->nvstore.size = blockif_size(sc->nvstore.ctx);
1742                 } else {
1743                         fprintf(stderr, "Invalid option %s\n", xopts);
1744                         return (-1);
1745                 }
1746
1747                 optidx++;
1748         }
1749         if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) {
1750                 fprintf(stderr, "backing store not specified\n");
1751                 return (-1);
1752         }
1753         if (sectsz == 512 || sectsz == 4096 || sectsz == 8192)
1754                 sc->nvstore.sectsz = sectsz;
1755         else if (sc->nvstore.type != NVME_STOR_RAM)
1756                 sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx);
1757         for (sc->nvstore.sectsz_bits = 9;
1758              (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;
1759              sc->nvstore.sectsz_bits++)
1760                 ;
1761
1762
1763         if (sc->max_queues == 0) {
1764                 fprintf(stderr, "Invalid maxq option\n");
1765                 return (-1);
1766         }
1767         if (sc->max_qentries <= 0) {
1768                 fprintf(stderr, "Invalid qsz option\n");
1769                 return (-1);
1770         }
1771         if (sc->ioslots <= 0) {
1772                 fprintf(stderr, "Invalid ioslots option\n");
1773                 return (-1);
1774         }
1775
1776         return (0);
1777 }
1778
1779 static int
1780 pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
1781 {
1782         struct pci_nvme_softc *sc;
1783         uint32_t pci_membar_sz;
1784         int     error;
1785
1786         error = 0;
1787
1788         sc = calloc(1, sizeof(struct pci_nvme_softc));
1789         pi->pi_arg = sc;
1790         sc->nsc_pi = pi;
1791
1792         error = pci_nvme_parse_opts(sc, opts);
1793         if (error < 0)
1794                 goto done;
1795         else
1796                 error = 0;
1797
1798         sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq));
1799         for (int i = 0; i < sc->ioslots; i++) {
1800                 if (i < (sc->ioslots-1))
1801                         sc->ioreqs[i].next = &sc->ioreqs[i+1];
1802                 pthread_mutex_init(&sc->ioreqs[i].mtx, NULL);
1803                 pthread_cond_init(&sc->ioreqs[i].cv, NULL);
1804         }
1805         sc->ioreqs_free = sc->ioreqs;
1806         sc->intr_coales_aggr_thresh = 1;
1807
1808         pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A);
1809         pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D);
1810         pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
1811         pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM);
1812         pci_set_cfgdata8(pi, PCIR_PROGIF,
1813                          PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0);
1814
1815         /* allocate size of nvme registers + doorbell space for all queues */
1816         pci_membar_sz = sizeof(struct nvme_registers) +
1817                         2*sizeof(uint32_t)*(sc->max_queues);
1818
1819         DPRINTF(("nvme membar size: %u\r\n", pci_membar_sz));
1820
1821         error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz);
1822         if (error) {
1823                 WPRINTF(("%s pci alloc mem bar failed\r\n", __func__));
1824                 goto done;
1825         }
1826
1827         error = pci_emul_add_msixcap(pi, sc->max_queues, NVME_MSIX_BAR);
1828         if (error) {
1829                 WPRINTF(("%s pci add msixcap failed\r\n", __func__));
1830                 goto done;
1831         }
1832
1833         pthread_mutex_init(&sc->mtx, NULL);
1834         sem_init(&sc->iosemlock, 0, sc->ioslots);
1835
1836         pci_nvme_reset(sc);
1837         pci_nvme_init_ctrldata(sc);
1838         pci_nvme_init_nsdata(sc);
1839
1840         pci_lintr_request(pi);
1841
1842 done:
1843         return (error);
1844 }
1845
1846
1847 struct pci_devemu pci_de_nvme = {
1848         .pe_emu =       "nvme",
1849         .pe_init =      pci_nvme_init,
1850         .pe_barwrite =  pci_nvme_write,
1851         .pe_barread =   pci_nvme_read
1852 };
1853 PCI_EMUL_SET(pci_de_nvme);