sys/dev/nvme/nvme.c

   1 /*-
   2  * Copyright (C) 2012 Intel Corporation
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/bus.h>
  32 #include <sys/conf.h>
  33 #include <sys/module.h>
  34
  35 #include <dev/pci/pcivar.h>
  36
  37 #include "nvme_private.h"
  38
  39 struct nvme_consumer {
  40         nvme_consumer_cb_fn_t           cb_fn;
  41         void                            *cb_arg;
  42 };
  43
  44 struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS];
  45
  46 MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations");
  47
  48 static int    nvme_probe(device_t);
  49 static int    nvme_attach(device_t);
  50 static int    nvme_detach(device_t);
  51
  52 static devclass_t nvme_devclass;
  53
  54 static device_method_t nvme_pci_methods[] = {
  55         /* Device interface */
  56         DEVMETHOD(device_probe,     nvme_probe),
  57         DEVMETHOD(device_attach,    nvme_attach),
  58         DEVMETHOD(device_detach,    nvme_detach),
  59         { 0, 0 }
  60 };
  61
  62 static driver_t nvme_pci_driver = {
  63         "nvme",
  64         nvme_pci_methods,
  65         sizeof(struct nvme_controller),
  66 };
  67
  68 DRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, 0, 0);
  69 MODULE_VERSION(nvme, 1);
  70
  71 static struct _pcsid
  72 {
  73         u_int32_t   type;
  74         const char  *desc;
  75 } pci_ids[] = {
  76         { 0x01118086,           "NVMe Controller"  },
  77         { CHATHAM_PCI_ID,       "Chatham Prototype NVMe Controller"  },
  78         { IDT_PCI_ID,           "IDT NVMe Controller"  },
  79         { 0x00000000,           NULL  }
  80 };
  81
  82 static int
  83 nvme_probe (device_t device)
  84 {
  85         u_int32_t type = pci_get_devid(device);
  86         struct _pcsid *ep = pci_ids;
  87
  88         while (ep->type && ep->type != type)
  89                 ++ep;
  90
  91         if (ep->desc) {
  92                 device_set_desc(device, ep->desc);
  93                 return (BUS_PROBE_DEFAULT);
  94         } else
  95                 return (ENXIO);
  96 }
  97
  98 static void
  99 nvme_load(void)
 100 {
 101 }
 102
 103 static void
 104 nvme_unload(void)
 105 {
 106 }
 107
 108 static void
 109 nvme_shutdown(void)
 110 {
 111         device_t                *devlist;
 112         struct nvme_controller  *ctrlr;
 113         union cc_register       cc;
 114         union csts_register     csts;
 115         int                     dev, devcount;
 116
 117         if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
 118                 return;
 119
 120         for (dev = 0; dev < devcount; dev++) {
 121                 /*
 122                  * Only notify controller of shutdown when a real shutdown is
 123                  *  in process, not when a module unload occurs.  It seems at
 124                  *  least some controllers (Chatham at least) don't let you
 125                  *  re-enable the controller after shutdown notification has
 126                  *  been received.
 127                  */
 128                 ctrlr = DEVICE2SOFTC(devlist[dev]);
 129                 cc.raw = nvme_mmio_read_4(ctrlr, cc);
 130                 cc.bits.shn = NVME_SHN_NORMAL;
 131                 nvme_mmio_write_4(ctrlr, cc, cc.raw);
 132                 csts.raw = nvme_mmio_read_4(ctrlr, csts);
 133                 while (csts.bits.shst != NVME_SHST_COMPLETE) {
 134                         DELAY(5);
 135                         csts.raw = nvme_mmio_read_4(ctrlr, csts);
 136                 }
 137         }
 138
 139         free(devlist, M_TEMP);
 140 }
 141
 142 static int
 143 nvme_modevent(module_t mod, int type, void *arg)
 144 {
 145
 146         switch (type) {
 147         case MOD_LOAD:
 148                 nvme_load();
 149                 break;
 150         case MOD_UNLOAD:
 151                 nvme_unload();
 152                 break;
 153         case MOD_SHUTDOWN:
 154                 nvme_shutdown();
 155                 break;
 156         default:
 157                 break;
 158         }
 159
 160         return (0);
 161 }
 162
 163 moduledata_t nvme_mod = {
 164         "nvme",
 165         (modeventhand_t)nvme_modevent,
 166         0
 167 };
 168
 169 DECLARE_MODULE(nvme, nvme_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
 170
 171 void
 172 nvme_dump_command(struct nvme_command *cmd)
 173 {
 174         printf("opc:%x f:%x r1:%x cid:%x nsid:%x r2:%x r3:%x "
 175             "mptr:%qx prp1:%qx prp2:%qx cdw:%x %x %x %x %x %x\n",
 176             cmd->opc, cmd->fuse, cmd->rsvd1, cmd->cid, cmd->nsid,
 177             cmd->rsvd2, cmd->rsvd3,
 178             (long long unsigned int)cmd->mptr,
 179             (long long unsigned int)cmd->prp1,
 180             (long long unsigned int)cmd->prp2,
 181             cmd->cdw10, cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14,
 182             cmd->cdw15);
 183 }
 184
 185 void
 186 nvme_dump_completion(struct nvme_completion *cpl)
 187 {
 188         printf("cdw0:%08x sqhd:%04x sqid:%04x "
 189             "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
 190             cpl->cdw0, cpl->sqhd, cpl->sqid,
 191             cpl->cid, cpl->p, cpl->sf_sc, cpl->sf_sct, cpl->sf_m,
 192             cpl->sf_dnr);
 193 }
 194
 195 void
 196 nvme_payload_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
 197 {
 198         struct nvme_tracker     *tr;
 199         struct nvme_qpair       *qpair;
 200         struct nvme_prp_list    *prp_list;
 201         uint32_t                cur_nseg;
 202
 203         KASSERT(error == 0, ("nvme_payload_map error != 0\n"));
 204
 205         tr = (struct nvme_tracker *)arg;
 206         qpair = tr->qpair;
 207
 208         /*
 209          * Note that we specified PAGE_SIZE for alignment and max
 210          *  segment size when creating the bus dma tags.  So here
 211          *  we can safely just transfer each segment to its
 212          *  associated PRP entry.
 213          */
 214         tr->cmd.prp1 = seg[0].ds_addr;
 215
 216         if (nseg == 2) {
 217                 tr->cmd.prp2 = seg[1].ds_addr;
 218         } else if (nseg > 2) {
 219                 KASSERT(tr->prp_list,
 220                     ("prp_list needed but not attached to tracker\n"));
 221                 cur_nseg = 1;
 222                 prp_list = tr->prp_list;
 223                 tr->cmd.prp2 = (uint64_t)prp_list->bus_addr;
 224                 while (cur_nseg < nseg) {
 225                         prp_list->prp[cur_nseg-1] =
 226                             (uint64_t)seg[cur_nseg].ds_addr;
 227                         cur_nseg++;
 228                 }
 229         }
 230
 231         nvme_qpair_submit_cmd(qpair, tr);
 232 }
 233
 234 struct nvme_tracker *
 235 nvme_allocate_tracker(struct nvme_controller *ctrlr, boolean_t is_admin,
 236     nvme_cb_fn_t cb_fn, void *cb_arg, uint32_t payload_size, void *payload)
 237 {
 238         struct nvme_tracker     *tr;
 239         struct nvme_qpair       *qpair;
 240         uint32_t                modulo, offset, num_prps;
 241         boolean_t               alloc_prp_list = FALSE;
 242
 243         if (is_admin) {
 244                 qpair = &ctrlr->adminq;
 245         } else {
 246                 if (ctrlr->per_cpu_io_queues)
 247                         qpair = &ctrlr->ioq[curcpu];
 248                 else
 249                         qpair = &ctrlr->ioq[0];
 250         }
 251
 252         num_prps = payload_size / PAGE_SIZE;
 253         modulo = payload_size % PAGE_SIZE;
 254         offset = (uint32_t)((uintptr_t)payload % PAGE_SIZE);
 255
 256         if (modulo || offset)
 257                 num_prps += 1 + (modulo + offset - 1) / PAGE_SIZE;
 258
 259         if (num_prps > 2)
 260                 alloc_prp_list = TRUE;
 261
 262         tr = nvme_qpair_allocate_tracker(qpair, alloc_prp_list);
 263
 264         memset(&tr->cmd, 0, sizeof(tr->cmd));
 265
 266         tr->qpair = qpair;
 267         tr->cb_fn = cb_fn;
 268         tr->cb_arg = cb_arg;
 269         tr->payload_size = payload_size;
 270
 271         return (tr);
 272 }
 273
 274 static int
 275 nvme_attach(device_t dev)
 276 {
 277         struct nvme_controller  *ctrlr = DEVICE2SOFTC(dev);
 278         int                     status;
 279
 280         status = nvme_ctrlr_construct(ctrlr, dev);
 281
 282         if (status != 0)
 283                 return (status);
 284
 285         /*
 286          * Reset controller twice to ensure we do a transition from cc.en==1
 287          *  to cc.en==0.  This is because we don't really know what status
 288          *  the controller was left in when boot handed off to OS.
 289          */
 290         status = nvme_ctrlr_reset(ctrlr);
 291         if (status != 0)
 292                 return (status);
 293
 294         status = nvme_ctrlr_reset(ctrlr);
 295         if (status != 0)
 296                 return (status);
 297
 298         ctrlr->config_hook.ich_func = nvme_ctrlr_start;
 299         ctrlr->config_hook.ich_arg = ctrlr;
 300
 301         config_intrhook_establish(&ctrlr->config_hook);
 302
 303         return (0);
 304 }
 305
 306 static int
 307 nvme_detach (device_t dev)
 308 {
 309         struct nvme_controller  *ctrlr = DEVICE2SOFTC(dev);
 310         struct nvme_namespace   *ns;
 311         int                     i;
 312
 313         if (ctrlr->taskqueue) {
 314                 taskqueue_drain(ctrlr->taskqueue, &ctrlr->task);
 315                 taskqueue_free(ctrlr->taskqueue);
 316         }
 317
 318         for (i = 0; i < NVME_MAX_NAMESPACES; i++) {
 319                 ns = &ctrlr->ns[i];
 320                 if (ns->cdev)
 321                         destroy_dev(ns->cdev);
 322         }
 323
 324         if (ctrlr->cdev)
 325                 destroy_dev(ctrlr->cdev);
 326
 327         for (i = 0; i < ctrlr->num_io_queues; i++) {
 328                 nvme_io_qpair_destroy(&ctrlr->ioq[i]);
 329         }
 330
 331         free(ctrlr->ioq, M_NVME);
 332
 333         nvme_admin_qpair_destroy(&ctrlr->adminq);
 334
 335         if (ctrlr->resource != NULL) {
 336                 bus_release_resource(dev, SYS_RES_MEMORY,
 337                     ctrlr->resource_id, ctrlr->resource);
 338         }
 339
 340 #ifdef CHATHAM2
 341         if (ctrlr->chatham_resource != NULL) {
 342                 bus_release_resource(dev, SYS_RES_MEMORY,
 343                     ctrlr->chatham_resource_id, ctrlr->chatham_resource);
 344         }
 345 #endif
 346
 347         if (ctrlr->tag)
 348                 bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
 349
 350         if (ctrlr->res)
 351                 bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
 352                     rman_get_rid(ctrlr->res), ctrlr->res);
 353
 354         if (ctrlr->msix_enabled)
 355                 pci_release_msi(dev);
 356
 357         return (0);
 358 }
 359
 360 static void
 361 nvme_notify_consumer(struct nvme_consumer *consumer)
 362 {
 363         device_t                *devlist;
 364         struct nvme_controller  *ctrlr;
 365         int                     dev, ns, devcount;
 366
 367         if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
 368                 return;
 369
 370         for (dev = 0; dev < devcount; dev++) {
 371                 ctrlr = DEVICE2SOFTC(devlist[dev]);
 372                 for (ns = 0; ns < ctrlr->cdata.nn; ns++)
 373                         (*consumer->cb_fn)(consumer->cb_arg, &ctrlr->ns[ns]);
 374         }
 375
 376         free(devlist, M_TEMP);
 377 }
 378
 379 struct nvme_consumer *
 380 nvme_register_consumer(nvme_consumer_cb_fn_t cb_fn, void *cb_arg)
 381 {
 382         int i;
 383
 384         /*
 385          * TODO: add locking around consumer registration.  Not an issue
 386          *  right now since we only have one nvme consumer - nvd(4).
 387          */
 388         for (i = 0; i < NVME_MAX_CONSUMERS; i++)
 389                 if (nvme_consumer[i].cb_fn == NULL) {
 390                         nvme_consumer[i].cb_fn = cb_fn;
 391                         nvme_consumer[i].cb_arg = cb_arg;
 392
 393                         nvme_notify_consumer(&nvme_consumer[i]);
 394                         return (&nvme_consumer[i]);
 395                 }
 396
 397         printf("nvme(4): consumer not registered - no slots available\n");
 398         return (NULL);
 399 }
 400
 401 void
 402 nvme_unregister_consumer(struct nvme_consumer *consumer)
 403 {
 404
 405         consumer->cb_fn = NULL;
 406         consumer->cb_arg = NULL;
 407 }
 408