sys/dev/nvme/nvme_ctrlr.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (C) 2012-2016 Intel Corporation
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 #include "opt_cam.h"
  31 #include "opt_nvme.h"
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/buf.h>
  36 #include <sys/bus.h>
  37 #include <sys/conf.h>
  38 #include <sys/ioccom.h>
  39 #include <sys/proc.h>
  40 #include <sys/smp.h>
  41 #include <sys/uio.h>
  42 #include <sys/sbuf.h>
  43 #include <sys/endian.h>
  44 #include <machine/stdarg.h>
  45 #include <vm/vm.h>
  46
  47 #include "nvme_private.h"
  48
  49 #define B4_CHK_RDY_DELAY_MS     2300            /* work around controller bug */
  50
  51 static void nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
  52                                                 struct nvme_async_event_request *aer);
  53
  54 static void
  55 nvme_ctrlr_barrier(struct nvme_controller *ctrlr, int flags)
  56 {
  57         bus_barrier(ctrlr->resource, 0, rman_get_size(ctrlr->resource), flags);
  58 }
  59
  60 static void
  61 nvme_ctrlr_devctl_log(struct nvme_controller *ctrlr, const char *type, const char *msg, ...)
  62 {
  63         struct sbuf sb;
  64         va_list ap;
  65         int error;
  66
  67         if (sbuf_new(&sb, NULL, 0, SBUF_AUTOEXTEND | SBUF_NOWAIT) == NULL)
  68                 return;
  69         sbuf_printf(&sb, "%s: ", device_get_nameunit(ctrlr->dev));
  70         va_start(ap, msg);
  71         sbuf_vprintf(&sb, msg, ap);
  72         va_end(ap);
  73         error = sbuf_finish(&sb);
  74         if (error == 0)
  75                 printf("%s\n", sbuf_data(&sb));
  76
  77         sbuf_clear(&sb);
  78         sbuf_printf(&sb, "name=\"%s\" reason=\"", device_get_nameunit(ctrlr->dev));
  79         va_start(ap, msg);
  80         sbuf_vprintf(&sb, msg, ap);
  81         va_end(ap);
  82         sbuf_printf(&sb, "\"");
  83         error = sbuf_finish(&sb);
  84         if (error == 0)
  85                 devctl_notify("nvme", "controller", type, sbuf_data(&sb));
  86         sbuf_delete(&sb);
  87 }
  88
  89 static int
  90 nvme_ctrlr_construct_admin_qpair(struct nvme_controller *ctrlr)
  91 {
  92         struct nvme_qpair       *qpair;
  93         uint32_t                num_entries;
  94         int                     error;
  95
  96         qpair = &ctrlr->adminq;
  97         qpair->id = 0;
  98         qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
  99         qpair->domain = ctrlr->domain;
 100
 101         num_entries = NVME_ADMIN_ENTRIES;
 102         TUNABLE_INT_FETCH("hw.nvme.admin_entries", &num_entries);
 103         /*
 104          * If admin_entries was overridden to an invalid value, revert it
 105          *  back to our default value.
 106          */
 107         if (num_entries < NVME_MIN_ADMIN_ENTRIES ||
 108             num_entries > NVME_MAX_ADMIN_ENTRIES) {
 109                 nvme_printf(ctrlr, "invalid hw.nvme.admin_entries=%d "
 110                     "specified\n", num_entries);
 111                 num_entries = NVME_ADMIN_ENTRIES;
 112         }
 113
 114         /*
 115          * The admin queue's max xfer size is treated differently than the
 116          *  max I/O xfer size.  16KB is sufficient here - maybe even less?
 117          */
 118         error = nvme_qpair_construct(qpair, num_entries, NVME_ADMIN_TRACKERS,
 119              ctrlr);
 120         return (error);
 121 }
 122
 123 #define QP(ctrlr, c)    ((c) * (ctrlr)->num_io_queues / mp_ncpus)
 124
 125 static int
 126 nvme_ctrlr_construct_io_qpairs(struct nvme_controller *ctrlr)
 127 {
 128         struct nvme_qpair       *qpair;
 129         uint32_t                cap_lo;
 130         uint16_t                mqes;
 131         int                     c, error, i, n;
 132         int                     num_entries, num_trackers, max_entries;
 133
 134         /*
 135          * NVMe spec sets a hard limit of 64K max entries, but devices may
 136          * specify a smaller limit, so we need to check the MQES field in the
 137          * capabilities register. We have to cap the number of entries to the
 138          * current stride allows for in BAR 0/1, otherwise the remainder entries
 139          * are inaccessible. MQES should reflect this, and this is just a
 140          * fail-safe.
 141          */
 142         max_entries =
 143             (rman_get_size(ctrlr->resource) - nvme_mmio_offsetof(doorbell[0])) /
 144             (1 << (ctrlr->dstrd + 1));
 145         num_entries = NVME_IO_ENTRIES;
 146         TUNABLE_INT_FETCH("hw.nvme.io_entries", &num_entries);
 147         cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
 148         mqes = NVME_CAP_LO_MQES(cap_lo);
 149         num_entries = min(num_entries, mqes + 1);
 150         num_entries = min(num_entries, max_entries);
 151
 152         num_trackers = NVME_IO_TRACKERS;
 153         TUNABLE_INT_FETCH("hw.nvme.io_trackers", &num_trackers);
 154
 155         num_trackers = max(num_trackers, NVME_MIN_IO_TRACKERS);
 156         num_trackers = min(num_trackers, NVME_MAX_IO_TRACKERS);
 157         /*
 158          * No need to have more trackers than entries in the submit queue.  Note
 159          * also that for a queue size of N, we can only have (N-1) commands
 160          * outstanding, hence the "-1" here.
 161          */
 162         num_trackers = min(num_trackers, (num_entries-1));
 163
 164         /*
 165          * Our best estimate for the maximum number of I/Os that we should
 166          * normally have in flight at one time. This should be viewed as a hint,
 167          * not a hard limit and will need to be revisited when the upper layers
 168          * of the storage system grows multi-queue support.
 169          */
 170         ctrlr->max_hw_pend_io = num_trackers * ctrlr->num_io_queues * 3 / 4;
 171
 172         ctrlr->ioq = malloc(ctrlr->num_io_queues * sizeof(struct nvme_qpair),
 173             M_NVME, M_ZERO | M_WAITOK);
 174
 175         for (i = c = n = 0; i < ctrlr->num_io_queues; i++, c += n) {
 176                 qpair = &ctrlr->ioq[i];
 177
 178                 /*
 179                  * Admin queue has ID=0. IO queues start at ID=1 -
 180                  *  hence the 'i+1' here.
 181                  */
 182                 qpair->id = i + 1;
 183                 if (ctrlr->num_io_queues > 1) {
 184                         /* Find number of CPUs served by this queue. */
 185                         for (n = 1; QP(ctrlr, c + n) == i; n++)
 186                                 ;
 187                         /* Shuffle multiple NVMe devices between CPUs. */
 188                         qpair->cpu = c + (device_get_unit(ctrlr->dev)+n/2) % n;
 189                         qpair->domain = pcpu_find(qpair->cpu)->pc_domain;
 190                 } else {
 191                         qpair->cpu = CPU_FFS(&cpuset_domain[ctrlr->domain]) - 1;
 192                         qpair->domain = ctrlr->domain;
 193                 }
 194
 195                 /*
 196                  * For I/O queues, use the controller-wide max_xfer_size
 197                  *  calculated in nvme_attach().
 198                  */
 199                 error = nvme_qpair_construct(qpair, num_entries, num_trackers,
 200                     ctrlr);
 201                 if (error)
 202                         return (error);
 203
 204                 /*
 205                  * Do not bother binding interrupts if we only have one I/O
 206                  *  interrupt thread for this controller.
 207                  */
 208                 if (ctrlr->num_io_queues > 1)
 209                         bus_bind_intr(ctrlr->dev, qpair->res, qpair->cpu);
 210         }
 211
 212         return (0);
 213 }
 214
 215 static void
 216 nvme_ctrlr_fail(struct nvme_controller *ctrlr)
 217 {
 218         int i;
 219
 220         /*
 221          * No need to disable queues before failing them. Failing is a superet
 222          * of disabling (though pedantically we'd abort the AERs silently with
 223          * a different error, though when we fail, that hardly matters).
 224          */
 225         ctrlr->is_failed = true;
 226         nvme_qpair_fail(&ctrlr->adminq);
 227         if (ctrlr->ioq != NULL) {
 228                 for (i = 0; i < ctrlr->num_io_queues; i++) {
 229                         nvme_qpair_fail(&ctrlr->ioq[i]);
 230                 }
 231         }
 232         nvme_notify_fail_consumers(ctrlr);
 233 }
 234
 235 void
 236 nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
 237     struct nvme_request *req)
 238 {
 239
 240         mtx_lock(&ctrlr->lock);
 241         STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);
 242         mtx_unlock(&ctrlr->lock);
 243         if (!ctrlr->is_dying)
 244                 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->fail_req_task);
 245 }
 246
 247 static void
 248 nvme_ctrlr_fail_req_task(void *arg, int pending)
 249 {
 250         struct nvme_controller  *ctrlr = arg;
 251         struct nvme_request     *req;
 252
 253         mtx_lock(&ctrlr->lock);
 254         while ((req = STAILQ_FIRST(&ctrlr->fail_req)) != NULL) {
 255                 STAILQ_REMOVE_HEAD(&ctrlr->fail_req, stailq);
 256                 mtx_unlock(&ctrlr->lock);
 257                 nvme_qpair_manual_complete_request(req->qpair, req,
 258                     NVME_SCT_GENERIC, NVME_SC_ABORTED_BY_REQUEST);
 259                 mtx_lock(&ctrlr->lock);
 260         }
 261         mtx_unlock(&ctrlr->lock);
 262 }
 263
 264 /*
 265  * Wait for RDY to change.
 266  *
 267  * Starts sleeping for 1us and geometrically increases it the longer we wait,
 268  * capped at 1ms.
 269  */
 270 static int
 271 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
 272 {
 273         int timeout = ticks + MSEC_2_TICKS(ctrlr->ready_timeout_in_ms);
 274         sbintime_t delta_t = SBT_1US;
 275         uint32_t csts;
 276
 277         while (1) {
 278                 csts = nvme_mmio_read_4(ctrlr, csts);
 279                 if (csts == NVME_GONE)          /* Hot unplug. */
 280                         return (ENXIO);
 281                 if (((csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK)
 282                     == desired_val)
 283                         break;
 284                 if (timeout - ticks < 0) {
 285                         nvme_printf(ctrlr, "controller ready did not become %d "
 286                             "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);
 287                         return (ENXIO);
 288                 }
 289
 290                 pause_sbt("nvmerdy", delta_t, 0, C_PREL(1));
 291                 delta_t = min(SBT_1MS, delta_t * 3 / 2);
 292         }
 293
 294         return (0);
 295 }
 296
 297 static int
 298 nvme_ctrlr_disable(struct nvme_controller *ctrlr)
 299 {
 300         uint32_t cc;
 301         uint32_t csts;
 302         uint8_t  en, rdy;
 303         int err;
 304
 305         cc = nvme_mmio_read_4(ctrlr, cc);
 306         csts = nvme_mmio_read_4(ctrlr, csts);
 307
 308         en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
 309         rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
 310
 311         /*
 312          * Per 3.1.5 in NVME 1.3 spec, transitioning CC.EN from 0 to 1
 313          * when CSTS.RDY is 1 or transitioning CC.EN from 1 to 0 when
 314          * CSTS.RDY is 0 "has undefined results" So make sure that CSTS.RDY
 315          * isn't the desired value. Short circuit if we're already disabled.
 316          */
 317         if (en == 0) {
 318                 /* Wait for RDY == 0 or timeout & fail */
 319                 if (rdy == 0)
 320                         return (0);
 321                 return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
 322         }
 323         if (rdy == 0) {
 324                 /* EN == 1, wait for  RDY == 1 or timeout & fail */
 325                 err = nvme_ctrlr_wait_for_ready(ctrlr, 1);
 326                 if (err != 0)
 327                         return (err);
 328         }
 329
 330         cc &= ~NVME_CC_REG_EN_MASK;
 331         nvme_mmio_write_4(ctrlr, cc, cc);
 332
 333         /*
 334          * A few drives have firmware bugs that freeze the drive if we access
 335          * the mmio too soon after we disable.
 336          */
 337         if (ctrlr->quirks & QUIRK_DELAY_B4_CHK_RDY)
 338                 pause("nvmeR", MSEC_2_TICKS(B4_CHK_RDY_DELAY_MS));
 339         return (nvme_ctrlr_wait_for_ready(ctrlr, 0));
 340 }
 341
 342 static int
 343 nvme_ctrlr_enable(struct nvme_controller *ctrlr)
 344 {
 345         uint32_t        cc;
 346         uint32_t        csts;
 347         uint32_t        aqa;
 348         uint32_t        qsize;
 349         uint8_t         en, rdy;
 350         int             err;
 351
 352         cc = nvme_mmio_read_4(ctrlr, cc);
 353         csts = nvme_mmio_read_4(ctrlr, csts);
 354
 355         en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
 356         rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
 357
 358         /*
 359          * See note in nvme_ctrlr_disable. Short circuit if we're already enabled.
 360          */
 361         if (en == 1) {
 362                 if (rdy == 1)
 363                         return (0);
 364                 return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
 365         }
 366
 367         /* EN == 0 already wait for RDY == 0 or timeout & fail */
 368         err = nvme_ctrlr_wait_for_ready(ctrlr, 0);
 369         if (err != 0)
 370                 return (err);
 371
 372         nvme_mmio_write_8(ctrlr, asq, ctrlr->adminq.cmd_bus_addr);
 373         nvme_mmio_write_8(ctrlr, acq, ctrlr->adminq.cpl_bus_addr);
 374
 375         /* acqs and asqs are 0-based. */
 376         qsize = ctrlr->adminq.num_entries - 1;
 377
 378         aqa = 0;
 379         aqa = (qsize & NVME_AQA_REG_ACQS_MASK) << NVME_AQA_REG_ACQS_SHIFT;
 380         aqa |= (qsize & NVME_AQA_REG_ASQS_MASK) << NVME_AQA_REG_ASQS_SHIFT;
 381         nvme_mmio_write_4(ctrlr, aqa, aqa);
 382
 383         /* Initialization values for CC */
 384         cc = 0;
 385         cc |= 1 << NVME_CC_REG_EN_SHIFT;
 386         cc |= 0 << NVME_CC_REG_CSS_SHIFT;
 387         cc |= 0 << NVME_CC_REG_AMS_SHIFT;
 388         cc |= 0 << NVME_CC_REG_SHN_SHIFT;
 389         cc |= 6 << NVME_CC_REG_IOSQES_SHIFT; /* SQ entry size == 64 == 2^6 */
 390         cc |= 4 << NVME_CC_REG_IOCQES_SHIFT; /* CQ entry size == 16 == 2^4 */
 391
 392         /*
 393          * Use the Memory Page Size selected during device initialization.  Note
 394          * that value stored in mps is suitable to use here without adjusting by
 395          * NVME_MPS_SHIFT.
 396          */
 397         cc |= ctrlr->mps << NVME_CC_REG_MPS_SHIFT;
 398
 399         nvme_ctrlr_barrier(ctrlr, BUS_SPACE_BARRIER_WRITE);
 400         nvme_mmio_write_4(ctrlr, cc, cc);
 401
 402         return (nvme_ctrlr_wait_for_ready(ctrlr, 1));
 403 }
 404
 405 static void
 406 nvme_ctrlr_disable_qpairs(struct nvme_controller *ctrlr)
 407 {
 408         int i;
 409
 410         nvme_admin_qpair_disable(&ctrlr->adminq);
 411         /*
 412          * I/O queues are not allocated before the initial HW
 413          *  reset, so do not try to disable them.  Use is_initialized
 414          *  to determine if this is the initial HW reset.
 415          */
 416         if (ctrlr->is_initialized) {
 417                 for (i = 0; i < ctrlr->num_io_queues; i++)
 418                         nvme_io_qpair_disable(&ctrlr->ioq[i]);
 419         }
 420 }
 421
 422 static int
 423 nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr)
 424 {
 425         int err;
 426
 427         TSENTER();
 428
 429         nvme_ctrlr_disable_qpairs(ctrlr);
 430
 431         err = nvme_ctrlr_disable(ctrlr);
 432         if (err != 0)
 433                 goto out;
 434
 435         err = nvme_ctrlr_enable(ctrlr);
 436 out:
 437
 438         TSEXIT();
 439         return (err);
 440 }
 441
 442 void
 443 nvme_ctrlr_reset(struct nvme_controller *ctrlr)
 444 {
 445         int cmpset;
 446
 447         cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
 448
 449         if (cmpset == 0 || ctrlr->is_failed)
 450                 /*
 451                  * Controller is already resetting or has failed.  Return
 452                  *  immediately since there is no need to kick off another
 453                  *  reset in these cases.
 454                  */
 455                 return;
 456
 457         if (!ctrlr->is_dying)
 458                 taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
 459 }
 460
 461 static int
 462 nvme_ctrlr_identify(struct nvme_controller *ctrlr)
 463 {
 464         struct nvme_completion_poll_status      status;
 465
 466         status.done = 0;
 467         nvme_ctrlr_cmd_identify_controller(ctrlr, &ctrlr->cdata,
 468             nvme_completion_poll_cb, &status);
 469         nvme_completion_poll(&status);
 470         if (nvme_completion_is_error(&status.cpl)) {
 471                 nvme_printf(ctrlr, "nvme_identify_controller failed!\n");
 472                 return (ENXIO);
 473         }
 474
 475         /* Convert data to host endian */
 476         nvme_controller_data_swapbytes(&ctrlr->cdata);
 477
 478         /*
 479          * Use MDTS to ensure our default max_xfer_size doesn't exceed what the
 480          *  controller supports.
 481          */
 482         if (ctrlr->cdata.mdts > 0)
 483                 ctrlr->max_xfer_size = min(ctrlr->max_xfer_size,
 484                     1 << (ctrlr->cdata.mdts + NVME_MPS_SHIFT +
 485                         NVME_CAP_HI_MPSMIN(ctrlr->cap_hi)));
 486
 487         return (0);
 488 }
 489
 490 static int
 491 nvme_ctrlr_set_num_qpairs(struct nvme_controller *ctrlr)
 492 {
 493         struct nvme_completion_poll_status      status;
 494         int                                     cq_allocated, sq_allocated;
 495
 496         status.done = 0;
 497         nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->num_io_queues,
 498             nvme_completion_poll_cb, &status);
 499         nvme_completion_poll(&status);
 500         if (nvme_completion_is_error(&status.cpl)) {
 501                 nvme_printf(ctrlr, "nvme_ctrlr_set_num_qpairs failed!\n");
 502                 return (ENXIO);
 503         }
 504
 505         /*
 506          * Data in cdw0 is 0-based.
 507          * Lower 16-bits indicate number of submission queues allocated.
 508          * Upper 16-bits indicate number of completion queues allocated.
 509          */
 510         sq_allocated = (status.cpl.cdw0 & 0xFFFF) + 1;
 511         cq_allocated = (status.cpl.cdw0 >> 16) + 1;
 512
 513         /*
 514          * Controller may allocate more queues than we requested,
 515          *  so use the minimum of the number requested and what was
 516          *  actually allocated.
 517          */
 518         ctrlr->num_io_queues = min(ctrlr->num_io_queues, sq_allocated);
 519         ctrlr->num_io_queues = min(ctrlr->num_io_queues, cq_allocated);
 520         if (ctrlr->num_io_queues > vm_ndomains)
 521                 ctrlr->num_io_queues -= ctrlr->num_io_queues % vm_ndomains;
 522
 523         return (0);
 524 }
 525
 526 static int
 527 nvme_ctrlr_create_qpairs(struct nvme_controller *ctrlr)
 528 {
 529         struct nvme_completion_poll_status      status;
 530         struct nvme_qpair                       *qpair;
 531         int                                     i;
 532
 533         for (i = 0; i < ctrlr->num_io_queues; i++) {
 534                 qpair = &ctrlr->ioq[i];
 535
 536                 status.done = 0;
 537                 nvme_ctrlr_cmd_create_io_cq(ctrlr, qpair,
 538                     nvme_completion_poll_cb, &status);
 539                 nvme_completion_poll(&status);
 540                 if (nvme_completion_is_error(&status.cpl)) {
 541                         nvme_printf(ctrlr, "nvme_create_io_cq failed!\n");
 542                         return (ENXIO);
 543                 }
 544
 545                 status.done = 0;
 546                 nvme_ctrlr_cmd_create_io_sq(ctrlr, qpair,
 547                     nvme_completion_poll_cb, &status);
 548                 nvme_completion_poll(&status);
 549                 if (nvme_completion_is_error(&status.cpl)) {
 550                         nvme_printf(ctrlr, "nvme_create_io_sq failed!\n");
 551                         return (ENXIO);
 552                 }
 553         }
 554
 555         return (0);
 556 }
 557
 558 static int
 559 nvme_ctrlr_delete_qpairs(struct nvme_controller *ctrlr)
 560 {
 561         struct nvme_completion_poll_status      status;
 562         struct nvme_qpair                       *qpair;
 563
 564         for (int i = 0; i < ctrlr->num_io_queues; i++) {
 565                 qpair = &ctrlr->ioq[i];
 566
 567                 status.done = 0;
 568                 nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair,
 569                     nvme_completion_poll_cb, &status);
 570                 nvme_completion_poll(&status);
 571                 if (nvme_completion_is_error(&status.cpl)) {
 572                         nvme_printf(ctrlr, "nvme_destroy_io_sq failed!\n");
 573                         return (ENXIO);
 574                 }
 575
 576                 status.done = 0;
 577                 nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair,
 578                     nvme_completion_poll_cb, &status);
 579                 nvme_completion_poll(&status);
 580                 if (nvme_completion_is_error(&status.cpl)) {
 581                         nvme_printf(ctrlr, "nvme_destroy_io_cq failed!\n");
 582                         return (ENXIO);
 583                 }
 584         }
 585
 586         return (0);
 587 }
 588
 589 static int
 590 nvme_ctrlr_construct_namespaces(struct nvme_controller *ctrlr)
 591 {
 592         struct nvme_namespace   *ns;
 593         uint32_t                i;
 594
 595         for (i = 0; i < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); i++) {
 596                 ns = &ctrlr->ns[i];
 597                 nvme_ns_construct(ns, i+1, ctrlr);
 598         }
 599
 600         return (0);
 601 }
 602
 603 static bool
 604 is_log_page_id_valid(uint8_t page_id)
 605 {
 606
 607         switch (page_id) {
 608         case NVME_LOG_ERROR:
 609         case NVME_LOG_HEALTH_INFORMATION:
 610         case NVME_LOG_FIRMWARE_SLOT:
 611         case NVME_LOG_CHANGED_NAMESPACE:
 612         case NVME_LOG_COMMAND_EFFECT:
 613         case NVME_LOG_RES_NOTIFICATION:
 614         case NVME_LOG_SANITIZE_STATUS:
 615                 return (true);
 616         }
 617
 618         return (false);
 619 }
 620
 621 static uint32_t
 622 nvme_ctrlr_get_log_page_size(struct nvme_controller *ctrlr, uint8_t page_id)
 623 {
 624         uint32_t        log_page_size;
 625
 626         switch (page_id) {
 627         case NVME_LOG_ERROR:
 628                 log_page_size = min(
 629                     sizeof(struct nvme_error_information_entry) *
 630                     (ctrlr->cdata.elpe + 1), NVME_MAX_AER_LOG_SIZE);
 631                 break;
 632         case NVME_LOG_HEALTH_INFORMATION:
 633                 log_page_size = sizeof(struct nvme_health_information_page);
 634                 break;
 635         case NVME_LOG_FIRMWARE_SLOT:
 636                 log_page_size = sizeof(struct nvme_firmware_page);
 637                 break;
 638         case NVME_LOG_CHANGED_NAMESPACE:
 639                 log_page_size = sizeof(struct nvme_ns_list);
 640                 break;
 641         case NVME_LOG_COMMAND_EFFECT:
 642                 log_page_size = sizeof(struct nvme_command_effects_page);
 643                 break;
 644         case NVME_LOG_RES_NOTIFICATION:
 645                 log_page_size = sizeof(struct nvme_res_notification_page);
 646                 break;
 647         case NVME_LOG_SANITIZE_STATUS:
 648                 log_page_size = sizeof(struct nvme_sanitize_status_page);
 649                 break;
 650         default:
 651                 log_page_size = 0;
 652                 break;
 653         }
 654
 655         return (log_page_size);
 656 }
 657
 658 static void
 659 nvme_ctrlr_log_critical_warnings(struct nvme_controller *ctrlr,
 660     uint8_t state)
 661 {
 662
 663         if (state & NVME_CRIT_WARN_ST_AVAILABLE_SPARE)
 664                 nvme_ctrlr_devctl_log(ctrlr, "critical",
 665                     "available spare space below threshold");
 666
 667         if (state & NVME_CRIT_WARN_ST_TEMPERATURE)
 668                 nvme_ctrlr_devctl_log(ctrlr, "critical",
 669                     "temperature above threshold");
 670
 671         if (state & NVME_CRIT_WARN_ST_DEVICE_RELIABILITY)
 672                 nvme_ctrlr_devctl_log(ctrlr, "critical",
 673                     "device reliability degraded");
 674
 675         if (state & NVME_CRIT_WARN_ST_READ_ONLY)
 676                 nvme_ctrlr_devctl_log(ctrlr, "critical",
 677                     "media placed in read only mode");
 678
 679         if (state & NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP)
 680                 nvme_ctrlr_devctl_log(ctrlr, "critical",
 681                     "volatile memory backup device failed");
 682
 683         if (state & NVME_CRIT_WARN_ST_RESERVED_MASK)
 684                 nvme_ctrlr_devctl_log(ctrlr, "critical",
 685                     "unknown critical warning(s): state = 0x%02x", state);
 686 }
 687
 688 static void
 689 nvme_ctrlr_async_event_log_page_cb(void *arg, const struct nvme_completion *cpl)
 690 {
 691         struct nvme_async_event_request         *aer = arg;
 692         struct nvme_health_information_page     *health_info;
 693         struct nvme_ns_list                     *nsl;
 694         struct nvme_error_information_entry     *err;
 695         int i;
 696
 697         /*
 698          * If the log page fetch for some reason completed with an error,
 699          *  don't pass log page data to the consumers.  In practice, this case
 700          *  should never happen.
 701          */
 702         if (nvme_completion_is_error(cpl))
 703                 nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
 704                     aer->log_page_id, NULL, 0);
 705         else {
 706                 /* Convert data to host endian */
 707                 switch (aer->log_page_id) {
 708                 case NVME_LOG_ERROR:
 709                         err = (struct nvme_error_information_entry *)aer->log_page_buffer;
 710                         for (i = 0; i < (aer->ctrlr->cdata.elpe + 1); i++)
 711                                 nvme_error_information_entry_swapbytes(err++);
 712                         break;
 713                 case NVME_LOG_HEALTH_INFORMATION:
 714                         nvme_health_information_page_swapbytes(
 715                             (struct nvme_health_information_page *)aer->log_page_buffer);
 716                         break;
 717                 case NVME_LOG_FIRMWARE_SLOT:
 718                         nvme_firmware_page_swapbytes(
 719                             (struct nvme_firmware_page *)aer->log_page_buffer);
 720                         break;
 721                 case NVME_LOG_CHANGED_NAMESPACE:
 722                         nvme_ns_list_swapbytes(
 723                             (struct nvme_ns_list *)aer->log_page_buffer);
 724                         break;
 725                 case NVME_LOG_COMMAND_EFFECT:
 726                         nvme_command_effects_page_swapbytes(
 727                             (struct nvme_command_effects_page *)aer->log_page_buffer);
 728                         break;
 729                 case NVME_LOG_RES_NOTIFICATION:
 730                         nvme_res_notification_page_swapbytes(
 731                             (struct nvme_res_notification_page *)aer->log_page_buffer);
 732                         break;
 733                 case NVME_LOG_SANITIZE_STATUS:
 734                         nvme_sanitize_status_page_swapbytes(
 735                             (struct nvme_sanitize_status_page *)aer->log_page_buffer);
 736                         break;
 737                 case INTEL_LOG_TEMP_STATS:
 738                         intel_log_temp_stats_swapbytes(
 739                             (struct intel_log_temp_stats *)aer->log_page_buffer);
 740                         break;
 741                 default:
 742                         break;
 743                 }
 744
 745                 if (aer->log_page_id == NVME_LOG_HEALTH_INFORMATION) {
 746                         health_info = (struct nvme_health_information_page *)
 747                             aer->log_page_buffer;
 748                         nvme_ctrlr_log_critical_warnings(aer->ctrlr,
 749                             health_info->critical_warning);
 750                         /*
 751                          * Critical warnings reported through the
 752                          *  SMART/health log page are persistent, so
 753                          *  clear the associated bits in the async event
 754                          *  config so that we do not receive repeated
 755                          *  notifications for the same event.
 756                          */
 757                         aer->ctrlr->async_event_config &=
 758                             ~health_info->critical_warning;
 759                         nvme_ctrlr_cmd_set_async_event_config(aer->ctrlr,
 760                             aer->ctrlr->async_event_config, NULL, NULL);
 761                 } else if (aer->log_page_id == NVME_LOG_CHANGED_NAMESPACE &&
 762                     !nvme_use_nvd) {
 763                         nsl = (struct nvme_ns_list *)aer->log_page_buffer;
 764                         for (i = 0; i < nitems(nsl->ns) && nsl->ns[i] != 0; i++) {
 765                                 if (nsl->ns[i] > NVME_MAX_NAMESPACES)
 766                                         break;
 767                                 nvme_notify_ns(aer->ctrlr, nsl->ns[i]);
 768                         }
 769                 }
 770
 771                 /*
 772                  * Pass the cpl data from the original async event completion,
 773                  *  not the log page fetch.
 774                  */
 775                 nvme_notify_async_consumers(aer->ctrlr, &aer->cpl,
 776                     aer->log_page_id, aer->log_page_buffer, aer->log_page_size);
 777         }
 778
 779         /*
 780          * Repost another asynchronous event request to replace the one
 781          *  that just completed.
 782          */
 783         nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
 784 }
 785
 786 static void
 787 nvme_ctrlr_async_event_cb(void *arg, const struct nvme_completion *cpl)
 788 {
 789         struct nvme_async_event_request *aer = arg;
 790
 791         if (nvme_completion_is_error(cpl)) {
 792                 /*
 793                  *  Do not retry failed async event requests.  This avoids
 794                  *  infinite loops where a new async event request is submitted
 795                  *  to replace the one just failed, only to fail again and
 796                  *  perpetuate the loop.
 797                  */
 798                 return;
 799         }
 800
 801         /* Associated log page is in bits 23:16 of completion entry dw0. */
 802         aer->log_page_id = (cpl->cdw0 & 0xFF0000) >> 16;
 803
 804         nvme_printf(aer->ctrlr, "async event occurred (type 0x%x, info 0x%02x,"
 805             " page 0x%02x)\n", (cpl->cdw0 & 0x07), (cpl->cdw0 & 0xFF00) >> 8,
 806             aer->log_page_id);
 807
 808         if (is_log_page_id_valid(aer->log_page_id)) {
 809                 aer->log_page_size = nvme_ctrlr_get_log_page_size(aer->ctrlr,
 810                     aer->log_page_id);
 811                 memcpy(&aer->cpl, cpl, sizeof(*cpl));
 812                 nvme_ctrlr_cmd_get_log_page(aer->ctrlr, aer->log_page_id,
 813                     NVME_GLOBAL_NAMESPACE_TAG, aer->log_page_buffer,
 814                     aer->log_page_size, nvme_ctrlr_async_event_log_page_cb,
 815                     aer);
 816                 /* Wait to notify consumers until after log page is fetched. */
 817         } else {
 818                 nvme_notify_async_consumers(aer->ctrlr, cpl, aer->log_page_id,
 819                     NULL, 0);
 820
 821                 /*
 822                  * Repost another asynchronous event request to replace the one
 823                  *  that just completed.
 824                  */
 825                 nvme_ctrlr_construct_and_submit_aer(aer->ctrlr, aer);
 826         }
 827 }
 828
 829 static void
 830 nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
 831     struct nvme_async_event_request *aer)
 832 {
 833         struct nvme_request *req;
 834
 835         aer->ctrlr = ctrlr;
 836         req = nvme_allocate_request_null(nvme_ctrlr_async_event_cb, aer);
 837         aer->req = req;
 838
 839         /*
 840          * Disable timeout here, since asynchronous event requests should by
 841          *  nature never be timed out.
 842          */
 843         req->timeout = false;
 844         req->cmd.opc = NVME_OPC_ASYNC_EVENT_REQUEST;
 845         nvme_ctrlr_submit_admin_request(ctrlr, req);
 846 }
 847
 848 static void
 849 nvme_ctrlr_configure_aer(struct nvme_controller *ctrlr)
 850 {
 851         struct nvme_completion_poll_status      status;
 852         struct nvme_async_event_request         *aer;
 853         uint32_t                                i;
 854
 855         ctrlr->async_event_config = NVME_CRIT_WARN_ST_AVAILABLE_SPARE |
 856             NVME_CRIT_WARN_ST_DEVICE_RELIABILITY |
 857             NVME_CRIT_WARN_ST_READ_ONLY |
 858             NVME_CRIT_WARN_ST_VOLATILE_MEMORY_BACKUP;
 859         if (ctrlr->cdata.ver >= NVME_REV(1, 2))
 860                 ctrlr->async_event_config |= NVME_ASYNC_EVENT_NS_ATTRIBUTE |
 861                     NVME_ASYNC_EVENT_FW_ACTIVATE;
 862
 863         status.done = 0;
 864         nvme_ctrlr_cmd_get_feature(ctrlr, NVME_FEAT_TEMPERATURE_THRESHOLD,
 865             0, NULL, 0, nvme_completion_poll_cb, &status);
 866         nvme_completion_poll(&status);
 867         if (nvme_completion_is_error(&status.cpl) ||
 868             (status.cpl.cdw0 & 0xFFFF) == 0xFFFF ||
 869             (status.cpl.cdw0 & 0xFFFF) == 0x0000) {
 870                 nvme_printf(ctrlr, "temperature threshold not supported\n");
 871         } else
 872                 ctrlr->async_event_config |= NVME_CRIT_WARN_ST_TEMPERATURE;
 873
 874         nvme_ctrlr_cmd_set_async_event_config(ctrlr,
 875             ctrlr->async_event_config, NULL, NULL);
 876
 877         /* aerl is a zero-based value, so we need to add 1 here. */
 878         ctrlr->num_aers = min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl+1));
 879
 880         for (i = 0; i < ctrlr->num_aers; i++) {
 881                 aer = &ctrlr->aer[i];
 882                 nvme_ctrlr_construct_and_submit_aer(ctrlr, aer);
 883         }
 884 }
 885
 886 static void
 887 nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
 888 {
 889
 890         ctrlr->int_coal_time = 0;
 891         TUNABLE_INT_FETCH("hw.nvme.int_coal_time",
 892             &ctrlr->int_coal_time);
 893
 894         ctrlr->int_coal_threshold = 0;
 895         TUNABLE_INT_FETCH("hw.nvme.int_coal_threshold",
 896             &ctrlr->int_coal_threshold);
 897
 898         nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr, ctrlr->int_coal_time,
 899             ctrlr->int_coal_threshold, NULL, NULL);
 900 }
 901
 902 static void
 903 nvme_ctrlr_hmb_free(struct nvme_controller *ctrlr)
 904 {
 905         struct nvme_hmb_chunk *hmbc;
 906         int i;
 907
 908         if (ctrlr->hmb_desc_paddr) {
 909                 bus_dmamap_unload(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map);
 910                 bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
 911                     ctrlr->hmb_desc_map);
 912                 ctrlr->hmb_desc_paddr = 0;
 913         }
 914         if (ctrlr->hmb_desc_tag) {
 915                 bus_dma_tag_destroy(ctrlr->hmb_desc_tag);
 916                 ctrlr->hmb_desc_tag = NULL;
 917         }
 918         for (i = 0; i < ctrlr->hmb_nchunks; i++) {
 919                 hmbc = &ctrlr->hmb_chunks[i];
 920                 bus_dmamap_unload(ctrlr->hmb_tag, hmbc->hmbc_map);
 921                 bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
 922                     hmbc->hmbc_map);
 923         }
 924         ctrlr->hmb_nchunks = 0;
 925         if (ctrlr->hmb_tag) {
 926                 bus_dma_tag_destroy(ctrlr->hmb_tag);
 927                 ctrlr->hmb_tag = NULL;
 928         }
 929         if (ctrlr->hmb_chunks) {
 930                 free(ctrlr->hmb_chunks, M_NVME);
 931                 ctrlr->hmb_chunks = NULL;
 932         }
 933 }
 934
 935 static void
 936 nvme_ctrlr_hmb_alloc(struct nvme_controller *ctrlr)
 937 {
 938         struct nvme_hmb_chunk *hmbc;
 939         size_t pref, min, minc, size;
 940         int err, i;
 941         uint64_t max;
 942
 943         /* Limit HMB to 5% of RAM size per device by default. */
 944         max = (uint64_t)physmem * PAGE_SIZE / 20;
 945         TUNABLE_UINT64_FETCH("hw.nvme.hmb_max", &max);
 946
 947         /*
 948          * Units of Host Memory Buffer in the Identify info are always in terms
 949          * of 4k units.
 950          */
 951         min = (long long unsigned)ctrlr->cdata.hmmin * NVME_HMB_UNITS;
 952         if (max == 0 || max < min)
 953                 return;
 954         pref = MIN((long long unsigned)ctrlr->cdata.hmpre * NVME_HMB_UNITS, max);
 955         minc = MAX(ctrlr->cdata.hmminds * NVME_HMB_UNITS, ctrlr->page_size);
 956         if (min > 0 && ctrlr->cdata.hmmaxd > 0)
 957                 minc = MAX(minc, min / ctrlr->cdata.hmmaxd);
 958         ctrlr->hmb_chunk = pref;
 959
 960 again:
 961         /*
 962          * However, the chunk sizes, number of chunks, and alignment of chunks
 963          * are all based on the current MPS (ctrlr->page_size).
 964          */
 965         ctrlr->hmb_chunk = roundup2(ctrlr->hmb_chunk, ctrlr->page_size);
 966         ctrlr->hmb_nchunks = howmany(pref, ctrlr->hmb_chunk);
 967         if (ctrlr->cdata.hmmaxd > 0 && ctrlr->hmb_nchunks > ctrlr->cdata.hmmaxd)
 968                 ctrlr->hmb_nchunks = ctrlr->cdata.hmmaxd;
 969         ctrlr->hmb_chunks = malloc(sizeof(struct nvme_hmb_chunk) *
 970             ctrlr->hmb_nchunks, M_NVME, M_WAITOK);
 971         err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
 972             ctrlr->page_size, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
 973             ctrlr->hmb_chunk, 1, ctrlr->hmb_chunk, 0, NULL, NULL, &ctrlr->hmb_tag);
 974         if (err != 0) {
 975                 nvme_printf(ctrlr, "HMB tag create failed %d\n", err);
 976                 nvme_ctrlr_hmb_free(ctrlr);
 977                 return;
 978         }
 979
 980         for (i = 0; i < ctrlr->hmb_nchunks; i++) {
 981                 hmbc = &ctrlr->hmb_chunks[i];
 982                 if (bus_dmamem_alloc(ctrlr->hmb_tag,
 983                     (void **)&hmbc->hmbc_vaddr, BUS_DMA_NOWAIT,
 984                     &hmbc->hmbc_map)) {
 985                         nvme_printf(ctrlr, "failed to alloc HMB\n");
 986                         break;
 987                 }
 988                 if (bus_dmamap_load(ctrlr->hmb_tag, hmbc->hmbc_map,
 989                     hmbc->hmbc_vaddr, ctrlr->hmb_chunk, nvme_single_map,
 990                     &hmbc->hmbc_paddr, BUS_DMA_NOWAIT) != 0) {
 991                         bus_dmamem_free(ctrlr->hmb_tag, hmbc->hmbc_vaddr,
 992                             hmbc->hmbc_map);
 993                         nvme_printf(ctrlr, "failed to load HMB\n");
 994                         break;
 995                 }
 996                 bus_dmamap_sync(ctrlr->hmb_tag, hmbc->hmbc_map,
 997                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 998         }
 999
1000         if (i < ctrlr->hmb_nchunks && i * ctrlr->hmb_chunk < min &&
1001             ctrlr->hmb_chunk / 2 >= minc) {
1002                 ctrlr->hmb_nchunks = i;
1003                 nvme_ctrlr_hmb_free(ctrlr);
1004                 ctrlr->hmb_chunk /= 2;
1005                 goto again;
1006         }
1007         ctrlr->hmb_nchunks = i;
1008         if (ctrlr->hmb_nchunks * ctrlr->hmb_chunk < min) {
1009                 nvme_ctrlr_hmb_free(ctrlr);
1010                 return;
1011         }
1012
1013         size = sizeof(struct nvme_hmb_desc) * ctrlr->hmb_nchunks;
1014         err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
1015             16, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
1016             size, 1, size, 0, NULL, NULL, &ctrlr->hmb_desc_tag);
1017         if (err != 0) {
1018                 nvme_printf(ctrlr, "HMB desc tag create failed %d\n", err);
1019                 nvme_ctrlr_hmb_free(ctrlr);
1020                 return;
1021         }
1022         if (bus_dmamem_alloc(ctrlr->hmb_desc_tag,
1023             (void **)&ctrlr->hmb_desc_vaddr, BUS_DMA_WAITOK,
1024             &ctrlr->hmb_desc_map)) {
1025                 nvme_printf(ctrlr, "failed to alloc HMB desc\n");
1026                 nvme_ctrlr_hmb_free(ctrlr);
1027                 return;
1028         }
1029         if (bus_dmamap_load(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
1030             ctrlr->hmb_desc_vaddr, size, nvme_single_map,
1031             &ctrlr->hmb_desc_paddr, BUS_DMA_NOWAIT) != 0) {
1032                 bus_dmamem_free(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_vaddr,
1033                     ctrlr->hmb_desc_map);
1034                 nvme_printf(ctrlr, "failed to load HMB desc\n");
1035                 nvme_ctrlr_hmb_free(ctrlr);
1036                 return;
1037         }
1038
1039         for (i = 0; i < ctrlr->hmb_nchunks; i++) {
1040                 ctrlr->hmb_desc_vaddr[i].addr =
1041                     htole64(ctrlr->hmb_chunks[i].hmbc_paddr);
1042                 ctrlr->hmb_desc_vaddr[i].size = htole32(ctrlr->hmb_chunk / ctrlr->page_size);
1043         }
1044         bus_dmamap_sync(ctrlr->hmb_desc_tag, ctrlr->hmb_desc_map,
1045             BUS_DMASYNC_PREWRITE);
1046
1047         nvme_printf(ctrlr, "Allocated %lluMB host memory buffer\n",
1048             (long long unsigned)ctrlr->hmb_nchunks * ctrlr->hmb_chunk
1049             / 1024 / 1024);
1050 }
1051
1052 static void
1053 nvme_ctrlr_hmb_enable(struct nvme_controller *ctrlr, bool enable, bool memret)
1054 {
1055         struct nvme_completion_poll_status      status;
1056         uint32_t cdw11;
1057
1058         cdw11 = 0;
1059         if (enable)
1060                 cdw11 |= 1;
1061         if (memret)
1062                 cdw11 |= 2;
1063         status.done = 0;
1064         nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_HOST_MEMORY_BUFFER, cdw11,
1065             ctrlr->hmb_nchunks * ctrlr->hmb_chunk / ctrlr->page_size,
1066             ctrlr->hmb_desc_paddr, ctrlr->hmb_desc_paddr >> 32,
1067             ctrlr->hmb_nchunks, NULL, 0,
1068             nvme_completion_poll_cb, &status);
1069         nvme_completion_poll(&status);
1070         if (nvme_completion_is_error(&status.cpl))
1071                 nvme_printf(ctrlr, "nvme_ctrlr_hmb_enable failed!\n");
1072 }
1073
1074 static void
1075 nvme_ctrlr_start(void *ctrlr_arg, bool resetting)
1076 {
1077         struct nvme_controller *ctrlr = ctrlr_arg;
1078         uint32_t old_num_io_queues;
1079         int i;
1080
1081         TSENTER();
1082
1083         /*
1084          * Only reset adminq here when we are restarting the
1085          *  controller after a reset.  During initialization,
1086          *  we have already submitted admin commands to get
1087          *  the number of I/O queues supported, so cannot reset
1088          *  the adminq again here.
1089          */
1090         if (resetting) {
1091                 nvme_qpair_reset(&ctrlr->adminq);
1092                 nvme_admin_qpair_enable(&ctrlr->adminq);
1093         }
1094
1095         if (ctrlr->ioq != NULL) {
1096                 for (i = 0; i < ctrlr->num_io_queues; i++)
1097                         nvme_qpair_reset(&ctrlr->ioq[i]);
1098         }
1099
1100         /*
1101          * If it was a reset on initialization command timeout, just
1102          * return here, letting initialization code fail gracefully.
1103          */
1104         if (resetting && !ctrlr->is_initialized)
1105                 return;
1106
1107         if (resetting && nvme_ctrlr_identify(ctrlr) != 0) {
1108                 nvme_ctrlr_fail(ctrlr);
1109                 return;
1110         }
1111
1112         /*
1113          * The number of qpairs are determined during controller initialization,
1114          *  including using NVMe SET_FEATURES/NUMBER_OF_QUEUES to determine the
1115          *  HW limit.  We call SET_FEATURES again here so that it gets called
1116          *  after any reset for controllers that depend on the driver to
1117          *  explicit specify how many queues it will use.  This value should
1118          *  never change between resets, so panic if somehow that does happen.
1119          */
1120         if (resetting) {
1121                 old_num_io_queues = ctrlr->num_io_queues;
1122                 if (nvme_ctrlr_set_num_qpairs(ctrlr) != 0) {
1123                         nvme_ctrlr_fail(ctrlr);
1124                         return;
1125                 }
1126
1127                 if (old_num_io_queues != ctrlr->num_io_queues) {
1128                         panic("num_io_queues changed from %u to %u",
1129                               old_num_io_queues, ctrlr->num_io_queues);
1130                 }
1131         }
1132
1133         if (ctrlr->cdata.hmpre > 0 && ctrlr->hmb_nchunks == 0) {
1134                 nvme_ctrlr_hmb_alloc(ctrlr);
1135                 if (ctrlr->hmb_nchunks > 0)
1136                         nvme_ctrlr_hmb_enable(ctrlr, true, false);
1137         } else if (ctrlr->hmb_nchunks > 0)
1138                 nvme_ctrlr_hmb_enable(ctrlr, true, true);
1139
1140         if (nvme_ctrlr_create_qpairs(ctrlr) != 0) {
1141                 nvme_ctrlr_fail(ctrlr);
1142                 return;
1143         }
1144
1145         if (nvme_ctrlr_construct_namespaces(ctrlr) != 0) {
1146                 nvme_ctrlr_fail(ctrlr);
1147                 return;
1148         }
1149
1150         nvme_ctrlr_configure_aer(ctrlr);
1151         nvme_ctrlr_configure_int_coalescing(ctrlr);
1152
1153         for (i = 0; i < ctrlr->num_io_queues; i++)
1154                 nvme_io_qpair_enable(&ctrlr->ioq[i]);
1155         TSEXIT();
1156 }
1157
1158 void
1159 nvme_ctrlr_start_config_hook(void *arg)
1160 {
1161         struct nvme_controller *ctrlr = arg;
1162
1163         TSENTER();
1164
1165         /*
1166          * Don't call pre/post reset here. We've not yet created the qpairs,
1167          * haven't setup the ISRs, so there's no need to 'drain' them or
1168          * 'exclude' them.
1169          */
1170         if (nvme_ctrlr_hw_reset(ctrlr) != 0) {
1171 fail:
1172                 nvme_ctrlr_fail(ctrlr);
1173                 config_intrhook_disestablish(&ctrlr->config_hook);
1174                 return;
1175         }
1176
1177 #ifdef NVME_2X_RESET
1178         /*
1179          * Reset controller twice to ensure we do a transition from cc.en==1 to
1180          * cc.en==0.  This is because we don't really know what status the
1181          * controller was left in when boot handed off to OS.  Linux doesn't do
1182          * this, however, and when the controller is in state cc.en == 0, no
1183          * I/O can happen.
1184          */
1185         if (nvme_ctrlr_hw_reset(ctrlr) != 0)
1186                 goto fail;
1187 #endif
1188
1189         nvme_qpair_reset(&ctrlr->adminq);
1190         nvme_admin_qpair_enable(&ctrlr->adminq);
1191
1192         if (nvme_ctrlr_identify(ctrlr) == 0 &&
1193             nvme_ctrlr_set_num_qpairs(ctrlr) == 0 &&
1194             nvme_ctrlr_construct_io_qpairs(ctrlr) == 0)
1195                 nvme_ctrlr_start(ctrlr, false);
1196         else
1197                 goto fail;
1198
1199         nvme_sysctl_initialize_ctrlr(ctrlr);
1200         config_intrhook_disestablish(&ctrlr->config_hook);
1201
1202         ctrlr->is_initialized = 1;
1203         nvme_notify_new_controller(ctrlr);
1204         TSEXIT();
1205 }
1206
1207 static void
1208 nvme_ctrlr_reset_task(void *arg, int pending)
1209 {
1210         struct nvme_controller  *ctrlr = arg;
1211         int                     status;
1212
1213         nvme_ctrlr_devctl_log(ctrlr, "RESET", "resetting controller");
1214         status = nvme_ctrlr_hw_reset(ctrlr);
1215         if (status == 0)
1216                 nvme_ctrlr_start(ctrlr, true);
1217         else
1218                 nvme_ctrlr_fail(ctrlr);
1219
1220         atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
1221 }
1222
1223 /*
1224  * Poll all the queues enabled on the device for completion.
1225  */
1226 void
1227 nvme_ctrlr_poll(struct nvme_controller *ctrlr)
1228 {
1229         int i;
1230
1231         nvme_qpair_process_completions(&ctrlr->adminq);
1232
1233         for (i = 0; i < ctrlr->num_io_queues; i++)
1234                 if (ctrlr->ioq && ctrlr->ioq[i].cpl)
1235                         nvme_qpair_process_completions(&ctrlr->ioq[i]);
1236 }
1237
1238 /*
1239  * Poll the single-vector interrupt case: num_io_queues will be 1 and
1240  * there's only a single vector. While we're polling, we mask further
1241  * interrupts in the controller.
1242  */
1243 void
1244 nvme_ctrlr_shared_handler(void *arg)
1245 {
1246         struct nvme_controller *ctrlr = arg;
1247
1248         nvme_mmio_write_4(ctrlr, intms, 1);
1249         nvme_ctrlr_poll(ctrlr);
1250         nvme_mmio_write_4(ctrlr, intmc, 1);
1251 }
1252
1253 static void
1254 nvme_pt_done(void *arg, const struct nvme_completion *cpl)
1255 {
1256         struct nvme_pt_command *pt = arg;
1257         struct mtx *mtx = pt->driver_lock;
1258         uint16_t status;
1259
1260         bzero(&pt->cpl, sizeof(pt->cpl));
1261         pt->cpl.cdw0 = cpl->cdw0;
1262
1263         status = cpl->status;
1264         status &= ~NVME_STATUS_P_MASK;
1265         pt->cpl.status = status;
1266
1267         mtx_lock(mtx);
1268         pt->driver_lock = NULL;
1269         wakeup(pt);
1270         mtx_unlock(mtx);
1271 }
1272
1273 int
1274 nvme_ctrlr_passthrough_cmd(struct nvme_controller *ctrlr,
1275     struct nvme_pt_command *pt, uint32_t nsid, int is_user_buffer,
1276     int is_admin_cmd)
1277 {
1278         struct nvme_request     *req;
1279         struct mtx              *mtx;
1280         struct buf              *buf = NULL;
1281         int                     ret = 0;
1282
1283         if (pt->len > 0) {
1284                 if (pt->len > ctrlr->max_xfer_size) {
1285                         nvme_printf(ctrlr, "pt->len (%d) "
1286                             "exceeds max_xfer_size (%d)\n", pt->len,
1287                             ctrlr->max_xfer_size);
1288                         return EIO;
1289                 }
1290                 if (is_user_buffer) {
1291                         /*
1292                          * Ensure the user buffer is wired for the duration of
1293                          *  this pass-through command.
1294                          */
1295                         PHOLD(curproc);
1296                         buf = uma_zalloc(pbuf_zone, M_WAITOK);
1297                         buf->b_iocmd = pt->is_read ? BIO_READ : BIO_WRITE;
1298                         if (vmapbuf(buf, pt->buf, pt->len, 1) < 0) {
1299                                 ret = EFAULT;
1300                                 goto err;
1301                         }
1302                         req = nvme_allocate_request_vaddr(buf->b_data, pt->len,
1303                             nvme_pt_done, pt);
1304                 } else
1305                         req = nvme_allocate_request_vaddr(pt->buf, pt->len,
1306                             nvme_pt_done, pt);
1307         } else
1308                 req = nvme_allocate_request_null(nvme_pt_done, pt);
1309
1310         /* Assume user space already converted to little-endian */
1311         req->cmd.opc = pt->cmd.opc;
1312         req->cmd.fuse = pt->cmd.fuse;
1313         req->cmd.rsvd2 = pt->cmd.rsvd2;
1314         req->cmd.rsvd3 = pt->cmd.rsvd3;
1315         req->cmd.cdw10 = pt->cmd.cdw10;
1316         req->cmd.cdw11 = pt->cmd.cdw11;
1317         req->cmd.cdw12 = pt->cmd.cdw12;
1318         req->cmd.cdw13 = pt->cmd.cdw13;
1319         req->cmd.cdw14 = pt->cmd.cdw14;
1320         req->cmd.cdw15 = pt->cmd.cdw15;
1321
1322         req->cmd.nsid = htole32(nsid);
1323
1324         mtx = mtx_pool_find(mtxpool_sleep, pt);
1325         pt->driver_lock = mtx;
1326
1327         if (is_admin_cmd)
1328                 nvme_ctrlr_submit_admin_request(ctrlr, req);
1329         else
1330                 nvme_ctrlr_submit_io_request(ctrlr, req);
1331
1332         mtx_lock(mtx);
1333         while (pt->driver_lock != NULL)
1334                 mtx_sleep(pt, mtx, PRIBIO, "nvme_pt", 0);
1335         mtx_unlock(mtx);
1336
1337 err:
1338         if (buf != NULL) {
1339                 uma_zfree(pbuf_zone, buf);
1340                 PRELE(curproc);
1341         }
1342
1343         return (ret);
1344 }
1345
1346 static int
1347 nvme_ctrlr_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
1348     struct thread *td)
1349 {
1350         struct nvme_controller                  *ctrlr;
1351         struct nvme_pt_command                  *pt;
1352
1353         ctrlr = cdev->si_drv1;
1354
1355         switch (cmd) {
1356         case NVME_RESET_CONTROLLER:
1357                 nvme_ctrlr_reset(ctrlr);
1358                 break;
1359         case NVME_PASSTHROUGH_CMD:
1360                 pt = (struct nvme_pt_command *)arg;
1361                 return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, le32toh(pt->cmd.nsid),
1362                     1 /* is_user_buffer */, 1 /* is_admin_cmd */));
1363         case NVME_GET_NSID:
1364         {
1365                 struct nvme_get_nsid *gnsid = (struct nvme_get_nsid *)arg;
1366                 strncpy(gnsid->cdev, device_get_nameunit(ctrlr->dev),
1367                     sizeof(gnsid->cdev));
1368                 gnsid->cdev[sizeof(gnsid->cdev) - 1] = '\0';
1369                 gnsid->nsid = 0;
1370                 break;
1371         }
1372         case NVME_GET_MAX_XFER_SIZE:
1373                 *(uint64_t *)arg = ctrlr->max_xfer_size;
1374                 break;
1375         default:
1376                 return (ENOTTY);
1377         }
1378
1379         return (0);
1380 }
1381
1382 static struct cdevsw nvme_ctrlr_cdevsw = {
1383         .d_version =    D_VERSION,
1384         .d_flags =      0,
1385         .d_ioctl =      nvme_ctrlr_ioctl
1386 };
1387
1388 int
1389 nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev)
1390 {
1391         struct make_dev_args    md_args;
1392         uint32_t        cap_lo;
1393         uint32_t        cap_hi;
1394         uint32_t        to, vs, pmrcap;
1395         int             status, timeout_period;
1396
1397         ctrlr->dev = dev;
1398
1399         mtx_init(&ctrlr->lock, "nvme ctrlr lock", NULL, MTX_DEF);
1400         if (bus_get_domain(dev, &ctrlr->domain) != 0)
1401                 ctrlr->domain = 0;
1402
1403         ctrlr->cap_lo = cap_lo = nvme_mmio_read_4(ctrlr, cap_lo);
1404         if (bootverbose) {
1405                 device_printf(dev, "CapLo: 0x%08x: MQES %u%s%s%s%s, TO %u\n",
1406                     cap_lo, NVME_CAP_LO_MQES(cap_lo),
1407                     NVME_CAP_LO_CQR(cap_lo) ? ", CQR" : "",
1408                     NVME_CAP_LO_AMS(cap_lo) ? ", AMS" : "",
1409                     (NVME_CAP_LO_AMS(cap_lo) & 0x1) ? " WRRwUPC" : "",
1410                     (NVME_CAP_LO_AMS(cap_lo) & 0x2) ? " VS" : "",
1411                     NVME_CAP_LO_TO(cap_lo));
1412         }
1413         ctrlr->cap_hi = cap_hi = nvme_mmio_read_4(ctrlr, cap_hi);
1414         if (bootverbose) {
1415                 device_printf(dev, "CapHi: 0x%08x: DSTRD %u%s, CSS %x%s, "
1416                     "MPSMIN %u, MPSMAX %u%s%s\n", cap_hi,
1417                     NVME_CAP_HI_DSTRD(cap_hi),
1418                     NVME_CAP_HI_NSSRS(cap_hi) ? ", NSSRS" : "",
1419                     NVME_CAP_HI_CSS(cap_hi),
1420                     NVME_CAP_HI_BPS(cap_hi) ? ", BPS" : "",
1421                     NVME_CAP_HI_MPSMIN(cap_hi),
1422                     NVME_CAP_HI_MPSMAX(cap_hi),
1423                     NVME_CAP_HI_PMRS(cap_hi) ? ", PMRS" : "",
1424                     NVME_CAP_HI_CMBS(cap_hi) ? ", CMBS" : "");
1425         }
1426         if (bootverbose) {
1427                 vs = nvme_mmio_read_4(ctrlr, vs);
1428                 device_printf(dev, "Version: 0x%08x: %d.%d\n", vs,
1429                     NVME_MAJOR(vs), NVME_MINOR(vs));
1430         }
1431         if (bootverbose && NVME_CAP_HI_PMRS(cap_hi)) {
1432                 pmrcap = nvme_mmio_read_4(ctrlr, pmrcap);
1433                 device_printf(dev, "PMRCap: 0x%08x: BIR %u%s%s, PMRTU %u, "
1434                     "PMRWBM %x, PMRTO %u%s\n", pmrcap,
1435                     NVME_PMRCAP_BIR(pmrcap),
1436                     NVME_PMRCAP_RDS(pmrcap) ? ", RDS" : "",
1437                     NVME_PMRCAP_WDS(pmrcap) ? ", WDS" : "",
1438                     NVME_PMRCAP_PMRTU(pmrcap),
1439                     NVME_PMRCAP_PMRWBM(pmrcap),
1440                     NVME_PMRCAP_PMRTO(pmrcap),
1441                     NVME_PMRCAP_CMSS(pmrcap) ? ", CMSS" : "");
1442         }
1443
1444         ctrlr->dstrd = NVME_CAP_HI_DSTRD(cap_hi) + 2;
1445
1446         ctrlr->mps = NVME_CAP_HI_MPSMIN(cap_hi);
1447         ctrlr->page_size = 1 << (NVME_MPS_SHIFT + ctrlr->mps);
1448
1449         /* Get ready timeout value from controller, in units of 500ms. */
1450         to = NVME_CAP_LO_TO(cap_lo) + 1;
1451         ctrlr->ready_timeout_in_ms = to * 500;
1452
1453         timeout_period = NVME_DEFAULT_TIMEOUT_PERIOD;
1454         TUNABLE_INT_FETCH("hw.nvme.timeout_period", &timeout_period);
1455         timeout_period = min(timeout_period, NVME_MAX_TIMEOUT_PERIOD);
1456         timeout_period = max(timeout_period, NVME_MIN_TIMEOUT_PERIOD);
1457         ctrlr->timeout_period = timeout_period;
1458
1459         nvme_retry_count = NVME_DEFAULT_RETRY_COUNT;
1460         TUNABLE_INT_FETCH("hw.nvme.retry_count", &nvme_retry_count);
1461
1462         ctrlr->enable_aborts = 0;
1463         TUNABLE_INT_FETCH("hw.nvme.enable_aborts", &ctrlr->enable_aborts);
1464
1465         /* Cap transfers by the maximum addressable by page-sized PRP (4KB pages -> 2MB). */
1466         ctrlr->max_xfer_size = MIN(maxphys, (ctrlr->page_size / 8 * ctrlr->page_size));
1467         if (nvme_ctrlr_construct_admin_qpair(ctrlr) != 0)
1468                 return (ENXIO);
1469
1470         /*
1471          * Create 2 threads for the taskqueue. The reset thread will block when
1472          * it detects that the controller has failed until all I/O has been
1473          * failed up the stack. The fail_req task needs to be able to run in
1474          * this case to finish the request failure for some cases.
1475          *
1476          * We could partially solve this race by draining the failed requeust
1477          * queue before proceding to free the sim, though nothing would stop
1478          * new I/O from coming in after we do that drain, but before we reach
1479          * cam_sim_free, so this big hammer is used instead.
1480          */
1481         ctrlr->taskqueue = taskqueue_create("nvme_taskq", M_WAITOK,
1482             taskqueue_thread_enqueue, &ctrlr->taskqueue);
1483         taskqueue_start_threads(&ctrlr->taskqueue, 2, PI_DISK, "nvme taskq");
1484
1485         ctrlr->is_resetting = 0;
1486         ctrlr->is_initialized = 0;
1487         ctrlr->notification_sent = 0;
1488         TASK_INIT(&ctrlr->reset_task, 0, nvme_ctrlr_reset_task, ctrlr);
1489         TASK_INIT(&ctrlr->fail_req_task, 0, nvme_ctrlr_fail_req_task, ctrlr);
1490         STAILQ_INIT(&ctrlr->fail_req);
1491         ctrlr->is_failed = false;
1492
1493         make_dev_args_init(&md_args);
1494         md_args.mda_devsw = &nvme_ctrlr_cdevsw;
1495         md_args.mda_uid = UID_ROOT;
1496         md_args.mda_gid = GID_WHEEL;
1497         md_args.mda_mode = 0600;
1498         md_args.mda_unit = device_get_unit(dev);
1499         md_args.mda_si_drv1 = (void *)ctrlr;
1500         status = make_dev_s(&md_args, &ctrlr->cdev, "nvme%d",
1501             device_get_unit(dev));
1502         if (status != 0)
1503                 return (ENXIO);
1504
1505         return (0);
1506 }
1507
1508 void
1509 nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev)
1510 {
1511         int     gone, i;
1512
1513         ctrlr->is_dying = true;
1514
1515         if (ctrlr->resource == NULL)
1516                 goto nores;
1517         if (!mtx_initialized(&ctrlr->adminq.lock))
1518                 goto noadminq;
1519
1520         /*
1521          * Check whether it is a hot unplug or a clean driver detach.
1522          * If device is not there any more, skip any shutdown commands.
1523          */
1524         gone = (nvme_mmio_read_4(ctrlr, csts) == NVME_GONE);
1525         if (gone)
1526                 nvme_ctrlr_fail(ctrlr);
1527         else
1528                 nvme_notify_fail_consumers(ctrlr);
1529
1530         for (i = 0; i < NVME_MAX_NAMESPACES; i++)
1531                 nvme_ns_destruct(&ctrlr->ns[i]);
1532
1533         if (ctrlr->cdev)
1534                 destroy_dev(ctrlr->cdev);
1535
1536         if (ctrlr->is_initialized) {
1537                 if (!gone) {
1538                         if (ctrlr->hmb_nchunks > 0)
1539                                 nvme_ctrlr_hmb_enable(ctrlr, false, false);
1540                         nvme_ctrlr_delete_qpairs(ctrlr);
1541                 }
1542                 nvme_ctrlr_hmb_free(ctrlr);
1543         }
1544         if (ctrlr->ioq != NULL) {
1545                 for (i = 0; i < ctrlr->num_io_queues; i++)
1546                         nvme_io_qpair_destroy(&ctrlr->ioq[i]);
1547                 free(ctrlr->ioq, M_NVME);
1548         }
1549         nvme_admin_qpair_destroy(&ctrlr->adminq);
1550
1551         /*
1552          *  Notify the controller of a shutdown, even though this is due to
1553          *   a driver unload, not a system shutdown (this path is not invoked
1554          *   during shutdown).  This ensures the controller receives a
1555          *   shutdown notification in case the system is shutdown before
1556          *   reloading the driver.
1557          */
1558         if (!gone)
1559                 nvme_ctrlr_shutdown(ctrlr);
1560
1561         if (!gone)
1562                 nvme_ctrlr_disable(ctrlr);
1563
1564 noadminq:
1565         if (ctrlr->taskqueue)
1566                 taskqueue_free(ctrlr->taskqueue);
1567
1568         if (ctrlr->tag)
1569                 bus_teardown_intr(ctrlr->dev, ctrlr->res, ctrlr->tag);
1570
1571         if (ctrlr->res)
1572                 bus_release_resource(ctrlr->dev, SYS_RES_IRQ,
1573                     rman_get_rid(ctrlr->res), ctrlr->res);
1574
1575         if (ctrlr->bar4_resource != NULL) {
1576                 bus_release_resource(dev, SYS_RES_MEMORY,
1577                     ctrlr->bar4_resource_id, ctrlr->bar4_resource);
1578         }
1579
1580         bus_release_resource(dev, SYS_RES_MEMORY,
1581             ctrlr->resource_id, ctrlr->resource);
1582
1583 nores:
1584         mtx_destroy(&ctrlr->lock);
1585 }
1586
1587 void
1588 nvme_ctrlr_shutdown(struct nvme_controller *ctrlr)
1589 {
1590         uint32_t        cc;
1591         uint32_t        csts;
1592         int             timeout;
1593
1594         cc = nvme_mmio_read_4(ctrlr, cc);
1595         cc &= ~(NVME_CC_REG_SHN_MASK << NVME_CC_REG_SHN_SHIFT);
1596         cc |= NVME_SHN_NORMAL << NVME_CC_REG_SHN_SHIFT;
1597         nvme_mmio_write_4(ctrlr, cc, cc);
1598
1599         timeout = ticks + (ctrlr->cdata.rtd3e == 0 ? 5 * hz :
1600             ((uint64_t)ctrlr->cdata.rtd3e * hz + 999999) / 1000000);
1601         while (1) {
1602                 csts = nvme_mmio_read_4(ctrlr, csts);
1603                 if (csts == NVME_GONE)          /* Hot unplug. */
1604                         break;
1605                 if (NVME_CSTS_GET_SHST(csts) == NVME_SHST_COMPLETE)
1606                         break;
1607                 if (timeout - ticks < 0) {
1608                         nvme_printf(ctrlr, "shutdown timeout\n");
1609                         break;
1610                 }
1611                 pause("nvmeshut", 1);
1612         }
1613 }
1614
1615 void
1616 nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
1617     struct nvme_request *req)
1618 {
1619
1620         nvme_qpair_submit_request(&ctrlr->adminq, req);
1621 }
1622
1623 void
1624 nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
1625     struct nvme_request *req)
1626 {
1627         struct nvme_qpair       *qpair;
1628
1629         qpair = &ctrlr->ioq[QP(ctrlr, curcpu)];
1630         nvme_qpair_submit_request(qpair, req);
1631 }
1632
1633 device_t
1634 nvme_ctrlr_get_device(struct nvme_controller *ctrlr)
1635 {
1636
1637         return (ctrlr->dev);
1638 }
1639
1640 const struct nvme_controller_data *
1641 nvme_ctrlr_get_data(struct nvme_controller *ctrlr)
1642 {
1643
1644         return (&ctrlr->cdata);
1645 }
1646
1647 int
1648 nvme_ctrlr_suspend(struct nvme_controller *ctrlr)
1649 {
1650         int to = hz;
1651
1652         /*
1653          * Can't touch failed controllers, so it's already suspended.
1654          */
1655         if (ctrlr->is_failed)
1656                 return (0);
1657
1658         /*
1659          * We don't want the reset taskqueue running, since it does similar
1660          * things, so prevent it from running after we start. Wait for any reset
1661          * that may have been started to complete. The reset process we follow
1662          * will ensure that any new I/O will queue and be given to the hardware
1663          * after we resume (though there should be none).
1664          */
1665         while (atomic_cmpset_32(&ctrlr->is_resetting, 0, 1) == 0 && to-- > 0)
1666                 pause("nvmesusp", 1);
1667         if (to <= 0) {
1668                 nvme_printf(ctrlr,
1669                     "Competing reset task didn't finish. Try again later.\n");
1670                 return (EWOULDBLOCK);
1671         }
1672
1673         if (ctrlr->hmb_nchunks > 0)
1674                 nvme_ctrlr_hmb_enable(ctrlr, false, false);
1675
1676         /*
1677          * Per Section 7.6.2 of NVMe spec 1.4, to properly suspend, we need to
1678          * delete the hardware I/O queues, and then shutdown. This properly
1679          * flushes any metadata the drive may have stored so it can survive
1680          * having its power removed and prevents the unsafe shutdown count from
1681          * incriminating. Once we delete the qpairs, we have to disable them
1682          * before shutting down.
1683          */
1684         nvme_ctrlr_delete_qpairs(ctrlr);
1685         nvme_ctrlr_disable_qpairs(ctrlr);
1686         nvme_ctrlr_shutdown(ctrlr);
1687
1688         return (0);
1689 }
1690
1691 int
1692 nvme_ctrlr_resume(struct nvme_controller *ctrlr)
1693 {
1694
1695         /*
1696          * Can't touch failed controllers, so nothing to do to resume.
1697          */
1698         if (ctrlr->is_failed)
1699                 return (0);
1700
1701         if (nvme_ctrlr_hw_reset(ctrlr) != 0)
1702                 goto fail;
1703
1704         /*
1705          * Now that we've reset the hardware, we can restart the controller. Any
1706          * I/O that was pending is requeued. Any admin commands are aborted with
1707          * an error. Once we've restarted, take the controller out of reset.
1708          */
1709         nvme_ctrlr_start(ctrlr, true);
1710         (void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
1711
1712         return (0);
1713 fail:
1714         /*
1715          * Since we can't bring the controller out of reset, announce and fail
1716          * the controller. However, we have to return success for the resume
1717          * itself, due to questionable APIs.
1718          */
1719         nvme_printf(ctrlr, "Failed to reset on resume, failing.\n");
1720         nvme_ctrlr_fail(ctrlr);
1721         (void)atomic_cmpset_32(&ctrlr->is_resetting, 1, 0);
1722         return (0);
1723 }