sys/cam/nvme/nvme_da.c

   1 /*-
   2  * Copyright (c) 2015 Netflix, Inc
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification, immediately at the beginning of the file.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * Derived from ata_da.c:
  27  * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34
  35 #ifdef _KERNEL
  36 #include <sys/systm.h>
  37 #include <sys/kernel.h>
  38 #include <sys/bio.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/taskqueue.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/conf.h>
  44 #include <sys/devicestat.h>
  45 #include <sys/eventhandler.h>
  46 #include <sys/malloc.h>
  47 #include <sys/cons.h>
  48 #include <sys/proc.h>
  49 #include <sys/reboot.h>
  50 #include <geom/geom_disk.h>
  51 #endif /* _KERNEL */
  52
  53 #ifndef _KERNEL
  54 #include <stdio.h>
  55 #include <string.h>
  56 #endif /* _KERNEL */
  57
  58 #include <cam/cam.h>
  59 #include <cam/cam_ccb.h>
  60 #include <cam/cam_periph.h>
  61 #include <cam/cam_xpt_periph.h>
  62 #include <cam/cam_sim.h>
  63 #include <cam/cam_iosched.h>
  64
  65 #include <cam/nvme/nvme_all.h>
  66
  67 typedef enum {
  68         NDA_STATE_NORMAL
  69 } nda_state;
  70
  71 typedef enum {
  72         NDA_FLAG_OPEN           = 0x0001,
  73         NDA_FLAG_DIRTY          = 0x0002,
  74         NDA_FLAG_SCTX_INIT      = 0x0004,
  75 } nda_flags;
  76
  77 typedef enum {
  78         NDA_Q_4K   = 0x01,
  79         NDA_Q_NONE = 0x00,
  80 } nda_quirks;
  81
  82 #define NDA_Q_BIT_STRING        \
  83         "\020"                  \
  84         "\001Bit 0"
  85
  86 typedef enum {
  87         NDA_CCB_BUFFER_IO       = 0x01,
  88         NDA_CCB_DUMP            = 0x02,
  89         NDA_CCB_TRIM            = 0x03,
  90         NDA_CCB_TYPE_MASK       = 0x0F,
  91 } nda_ccb_state;
  92
  93 /* Offsets into our private area for storing information */
  94 #define ccb_state       ppriv_field0
  95 #define ccb_bp          ppriv_ptr1
  96
  97 struct trim_request {
  98         TAILQ_HEAD(, bio) bps;
  99 };
 100 struct nda_softc {
 101         struct   cam_iosched_softc *cam_iosched;
 102         int      outstanding_cmds;      /* Number of active commands */
 103         int      refcount;              /* Active xpt_action() calls */
 104         nda_state state;
 105         nda_flags flags;
 106         nda_quirks quirks;
 107         int      unmappedio;
 108         uint32_t  nsid;                 /* Namespace ID for this nda device */
 109         struct disk *disk;
 110         struct task             sysctl_task;
 111         struct sysctl_ctx_list  sysctl_ctx;
 112         struct sysctl_oid       *sysctl_tree;
 113         struct trim_request     trim_req;
 114 #ifdef CAM_IO_STATS
 115         struct sysctl_ctx_list  sysctl_stats_ctx;
 116         struct sysctl_oid       *sysctl_stats_tree;
 117         u_int   timeouts;
 118         u_int   errors;
 119         u_int   invalidations;
 120 #endif
 121 };
 122
 123 /* Need quirk table */
 124
 125 static  disk_strategy_t ndastrategy;
 126 static  dumper_t        ndadump;
 127 static  periph_init_t   ndainit;
 128 static  void            ndaasync(void *callback_arg, u_int32_t code,
 129                                 struct cam_path *path, void *arg);
 130 static  void            ndasysctlinit(void *context, int pending);
 131 static  periph_ctor_t   ndaregister;
 132 static  periph_dtor_t   ndacleanup;
 133 static  periph_start_t  ndastart;
 134 static  periph_oninv_t  ndaoninvalidate;
 135 static  void            ndadone(struct cam_periph *periph,
 136                                union ccb *done_ccb);
 137 static  int             ndaerror(union ccb *ccb, u_int32_t cam_flags,
 138                                 u_int32_t sense_flags);
 139 static void             ndashutdown(void *arg, int howto);
 140 static void             ndasuspend(void *arg);
 141
 142 #ifndef NDA_DEFAULT_SEND_ORDERED
 143 #define NDA_DEFAULT_SEND_ORDERED        1
 144 #endif
 145 #ifndef NDA_DEFAULT_TIMEOUT
 146 #define NDA_DEFAULT_TIMEOUT 30  /* Timeout in seconds */
 147 #endif
 148 #ifndef NDA_DEFAULT_RETRY
 149 #define NDA_DEFAULT_RETRY       4
 150 #endif
 151
 152
 153 //static int nda_retry_count = NDA_DEFAULT_RETRY;
 154 static int nda_send_ordered = NDA_DEFAULT_SEND_ORDERED;
 155 static int nda_default_timeout = NDA_DEFAULT_TIMEOUT;
 156
 157 /*
 158  * All NVMe media is non-rotational, so all nvme device instances
 159  * share this to implement the sysctl.
 160  */
 161 static int nda_rotating_media = 0;
 162
 163 static SYSCTL_NODE(_kern_cam, OID_AUTO, nda, CTLFLAG_RD, 0,
 164             "CAM Direct Access Disk driver");
 165
 166 static struct periph_driver ndadriver =
 167 {
 168         ndainit, "nda",
 169         TAILQ_HEAD_INITIALIZER(ndadriver.units), /* generation */ 0
 170 };
 171
 172 PERIPHDRIVER_DECLARE(nda, ndadriver);
 173
 174 static MALLOC_DEFINE(M_NVMEDA, "nvme_da", "nvme_da buffers");
 175
 176 /*
 177  * nice wrappers. Maybe these belong in nvme_all.c instead of
 178  * here, but this is the only place that uses these. Should
 179  * we ever grow another NVME periph, we should move them
 180  * all there wholesale.
 181  */
 182
 183 static void
 184 nda_nvme_flush(struct nda_softc *softc, struct ccb_nvmeio *nvmeio)
 185 {
 186         cam_fill_nvmeio(nvmeio,
 187             0,                  /* retries */
 188             ndadone,            /* cbfcnp */
 189             CAM_DIR_NONE,       /* flags */
 190             NULL,               /* data_ptr */
 191             0,                  /* dxfer_len */
 192             nda_default_timeout * 1000); /* timeout 5s */
 193         nvme_ns_flush_cmd(&nvmeio->cmd, softc->nsid);
 194 }
 195
 196 static void
 197 nda_nvme_trim(struct nda_softc *softc, struct ccb_nvmeio *nvmeio,
 198     void *payload, uint32_t num_ranges)
 199 {
 200         cam_fill_nvmeio(nvmeio,
 201             0,                  /* retries */
 202             ndadone,            /* cbfcnp */
 203             CAM_DIR_OUT,        /* flags */
 204             payload,            /* data_ptr */
 205             num_ranges * sizeof(struct nvme_dsm_range), /* dxfer_len */
 206             nda_default_timeout * 1000); /* timeout 5s */
 207         nvme_ns_trim_cmd(&nvmeio->cmd, softc->nsid, num_ranges);
 208 }
 209
 210 static void
 211 nda_nvme_write(struct nda_softc *softc, struct ccb_nvmeio *nvmeio,
 212     void *payload, uint64_t lba, uint32_t len, uint32_t count)
 213 {
 214         cam_fill_nvmeio(nvmeio,
 215             0,                  /* retries */
 216             ndadone,            /* cbfcnp */
 217             CAM_DIR_OUT,        /* flags */
 218             payload,            /* data_ptr */
 219             len,                /* dxfer_len */
 220             nda_default_timeout * 1000); /* timeout 5s */
 221         nvme_ns_write_cmd(&nvmeio->cmd, softc->nsid, lba, count);
 222 }
 223
 224 static void
 225 nda_nvme_rw_bio(struct nda_softc *softc, struct ccb_nvmeio *nvmeio,
 226     struct bio *bp, uint32_t rwcmd)
 227 {
 228         int flags = rwcmd == NVME_OPC_READ ? CAM_DIR_IN : CAM_DIR_OUT;
 229         void *payload;
 230         uint64_t lba;
 231         uint32_t count;
 232
 233         if (bp->bio_flags & BIO_UNMAPPED) {
 234                 flags |= CAM_DATA_BIO;
 235                 payload = bp;
 236         } else {
 237                 payload = bp->bio_data;
 238         }
 239
 240         lba = bp->bio_pblkno;
 241         count = bp->bio_bcount / softc->disk->d_sectorsize;
 242
 243         cam_fill_nvmeio(nvmeio,
 244             0,                  /* retries */
 245             ndadone,            /* cbfcnp */
 246             flags,              /* flags */
 247             payload,            /* data_ptr */
 248             bp->bio_bcount,     /* dxfer_len */
 249             nda_default_timeout * 1000);                /* timeout 5s */
 250         nvme_ns_rw_cmd(&nvmeio->cmd, rwcmd, softc->nsid, lba, count);
 251 }
 252
 253 static int
 254 ndaopen(struct disk *dp)
 255 {
 256         struct cam_periph *periph;
 257         struct nda_softc *softc;
 258         int error;
 259
 260         periph = (struct cam_periph *)dp->d_drv1;
 261         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 262                 return(ENXIO);
 263         }
 264
 265         cam_periph_lock(periph);
 266         if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
 267                 cam_periph_unlock(periph);
 268                 cam_periph_release(periph);
 269                 return (error);
 270         }
 271
 272         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 273             ("ndaopen\n"));
 274
 275         softc = (struct nda_softc *)periph->softc;
 276         softc->flags |= NDA_FLAG_OPEN;
 277
 278         cam_periph_unhold(periph);
 279         cam_periph_unlock(periph);
 280         return (0);
 281 }
 282
 283 static int
 284 ndaclose(struct disk *dp)
 285 {
 286         struct  cam_periph *periph;
 287         struct  nda_softc *softc;
 288         union ccb *ccb;
 289         int error;
 290
 291         periph = (struct cam_periph *)dp->d_drv1;
 292         softc = (struct nda_softc *)periph->softc;
 293         cam_periph_lock(periph);
 294
 295         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH,
 296             ("ndaclose\n"));
 297
 298         if ((softc->flags & NDA_FLAG_DIRTY) != 0 &&
 299             (periph->flags & CAM_PERIPH_INVALID) == 0 &&
 300             cam_periph_hold(periph, PRIBIO) == 0) {
 301
 302                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 303                 nda_nvme_flush(softc, &ccb->nvmeio);
 304                 error = cam_periph_runccb(ccb, ndaerror, /*cam_flags*/0,
 305                     /*sense_flags*/0, softc->disk->d_devstat);
 306
 307                 if (error != 0)
 308                         xpt_print(periph->path, "Synchronize cache failed\n");
 309                 else
 310                         softc->flags &= ~NDA_FLAG_DIRTY;
 311                 xpt_release_ccb(ccb);
 312                 cam_periph_unhold(periph);
 313         }
 314
 315         softc->flags &= ~NDA_FLAG_OPEN;
 316
 317         while (softc->refcount != 0)
 318                 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "ndaclose", 1);
 319         cam_periph_unlock(periph);
 320         cam_periph_release(periph);
 321         return (0);
 322 }
 323
 324 static void
 325 ndaschedule(struct cam_periph *periph)
 326 {
 327         struct nda_softc *softc = (struct nda_softc *)periph->softc;
 328
 329         if (softc->state != NDA_STATE_NORMAL)
 330                 return;
 331
 332         cam_iosched_schedule(softc->cam_iosched, periph);
 333 }
 334
 335 /*
 336  * Actually translate the requested transfer into one the physical driver
 337  * can understand.  The transfer is described by a buf and will include
 338  * only one physical transfer.
 339  */
 340 static void
 341 ndastrategy(struct bio *bp)
 342 {
 343         struct cam_periph *periph;
 344         struct nda_softc *softc;
 345
 346         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
 347         softc = (struct nda_softc *)periph->softc;
 348
 349         cam_periph_lock(periph);
 350
 351         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastrategy(%p)\n", bp));
 352
 353         /*
 354          * If the device has been made invalid, error out
 355          */
 356         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 357                 cam_periph_unlock(periph);
 358                 biofinish(bp, NULL, ENXIO);
 359                 return;
 360         }
 361
 362         /*
 363          * Place it in the queue of disk activities for this disk
 364          */
 365         cam_iosched_queue_work(softc->cam_iosched, bp);
 366
 367         /*
 368          * Schedule ourselves for performing the work.
 369          */
 370         ndaschedule(periph);
 371         cam_periph_unlock(periph);
 372
 373         return;
 374 }
 375
 376 static int
 377 ndadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t length)
 378 {
 379         struct      cam_periph *periph;
 380         struct      nda_softc *softc;
 381         u_int       secsize;
 382         union       ccb ccb;
 383         struct      disk *dp;
 384         uint64_t    lba;
 385         uint32_t    count;
 386         int         error = 0;
 387
 388         dp = arg;
 389         periph = dp->d_drv1;
 390         softc = (struct nda_softc *)periph->softc;
 391         cam_periph_lock(periph);
 392         secsize = softc->disk->d_sectorsize;
 393         lba = offset / secsize;
 394         count = length / secsize;
 395
 396         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 397                 cam_periph_unlock(periph);
 398                 return (ENXIO);
 399         }
 400
 401         if (length > 0) {
 402                 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 403                 ccb.ccb_h.ccb_state = NDA_CCB_DUMP;
 404                 nda_nvme_write(softc, &ccb.nvmeio, virtual, lba, length, count);
 405                 xpt_polled_action(&ccb);
 406
 407                 error = cam_periph_error(&ccb,
 408                     0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
 409                 if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
 410                         cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
 411                             /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
 412                 if (error != 0)
 413                         printf("Aborting dump due to I/O error.\n");
 414
 415                 cam_periph_unlock(periph);
 416                 return (error);
 417         }
 418
 419         /* Flush */
 420         xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 421
 422         ccb.ccb_h.ccb_state = NDA_CCB_DUMP;
 423         nda_nvme_flush(softc, &ccb.nvmeio);
 424         xpt_polled_action(&ccb);
 425
 426         error = cam_periph_error(&ccb,
 427             0, SF_NO_RECOVERY | SF_NO_RETRY, NULL);
 428         if ((ccb.ccb_h.status & CAM_DEV_QFRZN) != 0)
 429                 cam_release_devq(ccb.ccb_h.path, /*relsim_flags*/0,
 430                     /*reduction*/0, /*timeout*/0, /*getcount_only*/0);
 431         if (error != 0)
 432                 xpt_print(periph->path, "flush cmd failed\n");
 433         cam_periph_unlock(periph);
 434         return (error);
 435 }
 436
 437 static void
 438 ndainit(void)
 439 {
 440         cam_status status;
 441
 442         /*
 443          * Install a global async callback.  This callback will
 444          * receive async callbacks like "new device found".
 445          */
 446         status = xpt_register_async(AC_FOUND_DEVICE, ndaasync, NULL, NULL);
 447
 448         if (status != CAM_REQ_CMP) {
 449                 printf("nda: Failed to attach master async callback "
 450                        "due to status 0x%x!\n", status);
 451         } else if (nda_send_ordered) {
 452
 453                 /* Register our event handlers */
 454                 if ((EVENTHANDLER_REGISTER(power_suspend, ndasuspend,
 455                                            NULL, EVENTHANDLER_PRI_LAST)) == NULL)
 456                     printf("ndainit: power event registration failed!\n");
 457                 if ((EVENTHANDLER_REGISTER(shutdown_post_sync, ndashutdown,
 458                                            NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
 459                     printf("ndainit: shutdown event registration failed!\n");
 460         }
 461 }
 462
 463 /*
 464  * Callback from GEOM, called when it has finished cleaning up its
 465  * resources.
 466  */
 467 static void
 468 ndadiskgonecb(struct disk *dp)
 469 {
 470         struct cam_periph *periph;
 471
 472         periph = (struct cam_periph *)dp->d_drv1;
 473
 474         cam_periph_release(periph);
 475 }
 476
 477 static void
 478 ndaoninvalidate(struct cam_periph *periph)
 479 {
 480         struct nda_softc *softc;
 481
 482         softc = (struct nda_softc *)periph->softc;
 483
 484         /*
 485          * De-register any async callbacks.
 486          */
 487         xpt_register_async(0, ndaasync, periph, periph->path);
 488 #ifdef CAM_IO_STATS
 489         softc->invalidations++;
 490 #endif
 491
 492         /*
 493          * Return all queued I/O with ENXIO.
 494          * XXX Handle any transactions queued to the card
 495          *     with XPT_ABORT_CCB.
 496          */
 497         cam_iosched_flush(softc->cam_iosched, NULL, ENXIO);
 498
 499         disk_gone(softc->disk);
 500 }
 501
 502 static void
 503 ndacleanup(struct cam_periph *periph)
 504 {
 505         struct nda_softc *softc;
 506
 507         softc = (struct nda_softc *)periph->softc;
 508
 509         cam_periph_unlock(periph);
 510
 511         cam_iosched_fini(softc->cam_iosched);
 512
 513         /*
 514          * If we can't free the sysctl tree, oh well...
 515          */
 516         if ((softc->flags & NDA_FLAG_SCTX_INIT) != 0) {
 517 #ifdef CAM_IO_STATS
 518                 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0)
 519                         xpt_print(periph->path,
 520                             "can't remove sysctl stats context\n");
 521 #endif
 522                 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0)
 523                         xpt_print(periph->path,
 524                             "can't remove sysctl context\n");
 525         }
 526
 527         disk_destroy(softc->disk);
 528         free(softc, M_DEVBUF);
 529         cam_periph_lock(periph);
 530 }
 531
 532 static void
 533 ndaasync(void *callback_arg, u_int32_t code,
 534         struct cam_path *path, void *arg)
 535 {
 536         struct cam_periph *periph;
 537
 538         periph = (struct cam_periph *)callback_arg;
 539         switch (code) {
 540         case AC_FOUND_DEVICE:
 541         {
 542                 struct ccb_getdev *cgd;
 543                 cam_status status;
 544
 545                 cgd = (struct ccb_getdev *)arg;
 546                 if (cgd == NULL)
 547                         break;
 548
 549                 if (cgd->protocol != PROTO_NVME)
 550                         break;
 551
 552                 /*
 553                  * Allocate a peripheral instance for
 554                  * this device and start the probe
 555                  * process.
 556                  */
 557                 status = cam_periph_alloc(ndaregister, ndaoninvalidate,
 558                                           ndacleanup, ndastart,
 559                                           "nda", CAM_PERIPH_BIO,
 560                                           path, ndaasync,
 561                                           AC_FOUND_DEVICE, cgd);
 562
 563                 if (status != CAM_REQ_CMP
 564                  && status != CAM_REQ_INPROG)
 565                         printf("ndaasync: Unable to attach to new device "
 566                                 "due to status 0x%x\n", status);
 567                 break;
 568         }
 569         case AC_ADVINFO_CHANGED:
 570         {
 571                 uintptr_t buftype;
 572
 573                 buftype = (uintptr_t)arg;
 574                 if (buftype == CDAI_TYPE_PHYS_PATH) {
 575                         struct nda_softc *softc;
 576
 577                         softc = periph->softc;
 578                         disk_attr_changed(softc->disk, "GEOM::physpath",
 579                                           M_NOWAIT);
 580                 }
 581                 break;
 582         }
 583         case AC_LOST_DEVICE:
 584         default:
 585                 cam_periph_async(periph, code, path, arg);
 586                 break;
 587         }
 588 }
 589
 590 static void
 591 ndasysctlinit(void *context, int pending)
 592 {
 593         struct cam_periph *periph;
 594         struct nda_softc *softc;
 595         char tmpstr[80], tmpstr2[80];
 596
 597         periph = (struct cam_periph *)context;
 598
 599         /* periph was held for us when this task was enqueued */
 600         if ((periph->flags & CAM_PERIPH_INVALID) != 0) {
 601                 cam_periph_release(periph);
 602                 return;
 603         }
 604
 605         softc = (struct nda_softc *)periph->softc;
 606         snprintf(tmpstr, sizeof(tmpstr), "CAM NDA unit %d", periph->unit_number);
 607         snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
 608
 609         sysctl_ctx_init(&softc->sysctl_ctx);
 610         softc->flags |= NDA_FLAG_SCTX_INIT;
 611         softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx,
 612                 SYSCTL_STATIC_CHILDREN(_kern_cam_nda), OID_AUTO, tmpstr2,
 613                 CTLFLAG_RD, 0, tmpstr, "device_index");
 614         if (softc->sysctl_tree == NULL) {
 615                 printf("ndasysctlinit: unable to allocate sysctl tree\n");
 616                 cam_periph_release(periph);
 617                 return;
 618         }
 619
 620         SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
 621                 OID_AUTO, "unmapped_io", CTLFLAG_RD | CTLFLAG_MPSAFE,
 622                 &softc->unmappedio, 0, "Unmapped I/O leaf");
 623
 624         SYSCTL_ADD_INT(&softc->sysctl_ctx,
 625                        SYSCTL_CHILDREN(softc->sysctl_tree),
 626                        OID_AUTO,
 627                        "rotating",
 628                        CTLFLAG_RD | CTLFLAG_MPSAFE,
 629                        &nda_rotating_media,
 630                        0,
 631                        "Rotating media");
 632
 633 #ifdef CAM_IO_STATS
 634         softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx,
 635                 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats",
 636                 CTLFLAG_RD, 0, "Statistics");
 637         if (softc->sysctl_stats_tree == NULL) {
 638                 printf("ndasysctlinit: unable to allocate sysctl tree for stats\n");
 639                 cam_periph_release(periph);
 640                 return;
 641         }
 642         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
 643                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
 644                 OID_AUTO, "timeouts", CTLFLAG_RD | CTLFLAG_MPSAFE,
 645                 &softc->timeouts, 0,
 646                 "Device timeouts reported by the SIM");
 647         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
 648                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
 649                 OID_AUTO, "errors", CTLFLAG_RD | CTLFLAG_MPSAFE,
 650                 &softc->errors, 0,
 651                 "Transport errors reported by the SIM.");
 652         SYSCTL_ADD_INT(&softc->sysctl_stats_ctx,
 653                 SYSCTL_CHILDREN(softc->sysctl_stats_tree),
 654                 OID_AUTO, "pack_invalidations", CTLFLAG_RD | CTLFLAG_MPSAFE,
 655                 &softc->invalidations, 0,
 656                 "Device pack invalidations.");
 657 #endif
 658
 659         cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx,
 660             softc->sysctl_tree);
 661
 662         cam_periph_release(periph);
 663 }
 664
 665 static int
 666 ndagetattr(struct bio *bp)
 667 {
 668         int ret;
 669         struct cam_periph *periph;
 670
 671         periph = (struct cam_periph *)bp->bio_disk->d_drv1;
 672         cam_periph_lock(periph);
 673         ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute,
 674             periph->path);
 675         cam_periph_unlock(periph);
 676         if (ret == 0)
 677                 bp->bio_completed = bp->bio_length;
 678         return ret;
 679 }
 680
 681 static cam_status
 682 ndaregister(struct cam_periph *periph, void *arg)
 683 {
 684         struct nda_softc *softc;
 685         struct disk *disk;
 686         struct ccb_pathinq cpi;
 687         struct ccb_getdev *cgd;
 688         const struct nvme_namespace_data *nsd;
 689         const struct nvme_controller_data *cd;
 690         char   announce_buf[80];
 691 //      caddr_t match;
 692         u_int maxio;
 693         int quirks;
 694
 695         cgd = (struct ccb_getdev *)arg;
 696         if (cgd == NULL) {
 697                 printf("ndaregister: no getdev CCB, can't register device\n");
 698                 return(CAM_REQ_CMP_ERR);
 699         }
 700         nsd = cgd->nvme_data;
 701         cd = cgd->nvme_cdata;
 702
 703         softc = (struct nda_softc *)malloc(sizeof(*softc), M_DEVBUF,
 704             M_NOWAIT | M_ZERO);
 705
 706         if (softc == NULL) {
 707                 printf("ndaregister: Unable to probe new device. "
 708                     "Unable to allocate softc\n");
 709                 return(CAM_REQ_CMP_ERR);
 710         }
 711
 712         if (cam_iosched_init(&softc->cam_iosched, periph) != 0) {
 713                 printf("ndaregister: Unable to probe new device. "
 714                        "Unable to allocate iosched memory\n");
 715                 return(CAM_REQ_CMP_ERR);
 716         }
 717
 718         /* ident_data parsing */
 719
 720         periph->softc = softc;
 721
 722 #if 0
 723         /*
 724          * See if this device has any quirks.
 725          */
 726         match = cam_quirkmatch((caddr_t)&cgd->ident_data,
 727                                (caddr_t)nda_quirk_table,
 728                                sizeof(nda_quirk_table)/sizeof(*nda_quirk_table),
 729                                sizeof(*nda_quirk_table), ata_identify_match);
 730         if (match != NULL)
 731                 softc->quirks = ((struct nda_quirk_entry *)match)->quirks;
 732         else
 733 #endif
 734                 softc->quirks = NDA_Q_NONE;
 735
 736         bzero(&cpi, sizeof(cpi));
 737         xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NONE);
 738         cpi.ccb_h.func_code = XPT_PATH_INQ;
 739         xpt_action((union ccb *)&cpi);
 740
 741         TASK_INIT(&softc->sysctl_task, 0, ndasysctlinit, periph);
 742
 743         /*
 744          * The name space ID is the lun, save it for later I/O
 745          */
 746         softc->nsid = (uint16_t)xpt_path_lun_id(periph->path);
 747
 748         /*
 749          * Register this media as a disk
 750          */
 751         (void)cam_periph_hold(periph, PRIBIO);
 752         cam_periph_unlock(periph);
 753         snprintf(announce_buf, sizeof(announce_buf),
 754             "kern.cam.nda.%d.quirks", periph->unit_number);
 755         quirks = softc->quirks;
 756         TUNABLE_INT_FETCH(announce_buf, &quirks);
 757         softc->quirks = quirks;
 758         cam_iosched_set_sort_queue(softc->cam_iosched, 0);
 759         softc->disk = disk = disk_alloc();
 760         strlcpy(softc->disk->d_descr, cd->mn,
 761             MIN(sizeof(softc->disk->d_descr), sizeof(cd->mn)));
 762         strlcpy(softc->disk->d_ident, cd->sn,
 763             MIN(sizeof(softc->disk->d_ident), sizeof(cd->sn)));
 764         disk->d_rotation_rate = DISK_RR_NON_ROTATING;
 765         disk->d_open = ndaopen;
 766         disk->d_close = ndaclose;
 767         disk->d_strategy = ndastrategy;
 768         disk->d_getattr = ndagetattr;
 769         disk->d_dump = ndadump;
 770         disk->d_gone = ndadiskgonecb;
 771         disk->d_name = "nda";
 772         disk->d_drv1 = periph;
 773         disk->d_unit = periph->unit_number;
 774         maxio = cpi.maxio;              /* Honor max I/O size of SIM */
 775         if (maxio == 0)
 776                 maxio = DFLTPHYS;       /* traditional default */
 777         else if (maxio > MAXPHYS)
 778                 maxio = MAXPHYS;        /* for safety */
 779         disk->d_maxsize = maxio;
 780         disk->d_sectorsize = 1 << nsd->lbaf[nsd->flbas.format].lbads;
 781         disk->d_mediasize = (off_t)(disk->d_sectorsize * nsd->nsze);
 782         disk->d_delmaxsize = disk->d_mediasize;
 783         disk->d_flags = DISKFLAG_DIRECT_COMPLETION;
 784 //      if (cd->oncs.dsm) // XXX broken?
 785                 disk->d_flags |= DISKFLAG_CANDELETE;
 786         if (cd->vwc.present)
 787                 disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
 788         if ((cpi.hba_misc & PIM_UNMAPPED) != 0) {
 789                 disk->d_flags |= DISKFLAG_UNMAPPED_BIO;
 790                 softc->unmappedio = 1;
 791         }
 792         /*
 793          * d_ident and d_descr are both far bigger than the length of either
 794          *  the serial or model number strings.
 795          */
 796         nvme_strvis(disk->d_descr, cd->mn,
 797             sizeof(disk->d_descr), NVME_MODEL_NUMBER_LENGTH);
 798         nvme_strvis(disk->d_ident, cd->sn,
 799             sizeof(disk->d_ident), NVME_SERIAL_NUMBER_LENGTH);
 800         disk->d_hba_vendor = cpi.hba_vendor;
 801         disk->d_hba_device = cpi.hba_device;
 802         disk->d_hba_subvendor = cpi.hba_subvendor;
 803         disk->d_hba_subdevice = cpi.hba_subdevice;
 804         disk->d_stripesize = disk->d_sectorsize;
 805         disk->d_stripeoffset = 0;
 806         disk->d_devstat = devstat_new_entry(periph->periph_name,
 807             periph->unit_number, disk->d_sectorsize,
 808             DEVSTAT_ALL_SUPPORTED,
 809             DEVSTAT_TYPE_DIRECT | XPORT_DEVSTAT_TYPE(cpi.transport),
 810             DEVSTAT_PRIORITY_DISK);
 811
 812         /*
 813          * Acquire a reference to the periph before we register with GEOM.
 814          * We'll release this reference once GEOM calls us back (via
 815          * ndadiskgonecb()) telling us that our provider has been freed.
 816          */
 817         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 818                 xpt_print(periph->path, "%s: lost periph during "
 819                           "registration!\n", __func__);
 820                 cam_periph_lock(periph);
 821                 return (CAM_REQ_CMP_ERR);
 822         }
 823         disk_create(softc->disk, DISK_VERSION);
 824         cam_periph_lock(periph);
 825         cam_periph_unhold(periph);
 826
 827         snprintf(announce_buf, sizeof(announce_buf),
 828                 "%juMB (%ju %u byte sectors)",
 829             (uintmax_t)((uintmax_t)disk->d_mediasize / (1024*1024)),
 830                 (uintmax_t)disk->d_mediasize / disk->d_sectorsize,
 831                 disk->d_sectorsize);
 832         xpt_announce_periph(periph, announce_buf);
 833         xpt_announce_quirks(periph, softc->quirks, NDA_Q_BIT_STRING);
 834
 835         /*
 836          * Create our sysctl variables, now that we know
 837          * we have successfully attached.
 838          */
 839         if (cam_periph_acquire(periph) == CAM_REQ_CMP)
 840                 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task);
 841
 842         /*
 843          * Register for device going away and info about the drive
 844          * changing (though with NVMe, it can't)
 845          */
 846         xpt_register_async(AC_LOST_DEVICE | AC_ADVINFO_CHANGED,
 847             ndaasync, periph, periph->path);
 848
 849         softc->state = NDA_STATE_NORMAL;
 850         return(CAM_REQ_CMP);
 851 }
 852
 853 static void
 854 ndastart(struct cam_periph *periph, union ccb *start_ccb)
 855 {
 856         struct nda_softc *softc = (struct nda_softc *)periph->softc;
 857         struct ccb_nvmeio *nvmeio = &start_ccb->nvmeio;
 858
 859         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart\n"));
 860
 861         switch (softc->state) {
 862         case NDA_STATE_NORMAL:
 863         {
 864                 struct bio *bp;
 865
 866                 bp = cam_iosched_next_bio(softc->cam_iosched);
 867                 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart: bio %p\n", bp));
 868                 if (bp == NULL) {
 869                         xpt_release_ccb(start_ccb);
 870                         break;
 871                 }
 872
 873                 switch (bp->bio_cmd) {
 874                 case BIO_WRITE:
 875                         softc->flags |= NDA_FLAG_DIRTY;
 876                         /* FALLTHROUGH */
 877                 case BIO_READ:
 878                 {
 879 #ifdef NDA_TEST_FAILURE
 880                         int fail = 0;
 881
 882                         /*
 883                          * Support the failure ioctls.  If the command is a
 884                          * read, and there are pending forced read errors, or
 885                          * if a write and pending write errors, then fail this
 886                          * operation with EIO.  This is useful for testing
 887                          * purposes.  Also, support having every Nth read fail.
 888                          *
 889                          * This is a rather blunt tool.
 890                          */
 891                         if (bp->bio_cmd == BIO_READ) {
 892                                 if (softc->force_read_error) {
 893                                         softc->force_read_error--;
 894                                         fail = 1;
 895                                 }
 896                                 if (softc->periodic_read_error > 0) {
 897                                         if (++softc->periodic_read_count >=
 898                                             softc->periodic_read_error) {
 899                                                 softc->periodic_read_count = 0;
 900                                                 fail = 1;
 901                                         }
 902                                 }
 903                         } else {
 904                                 if (softc->force_write_error) {
 905                                         softc->force_write_error--;
 906                                         fail = 1;
 907                                 }
 908                         }
 909                         if (fail) {
 910                                 biofinish(bp, NULL, EIO);
 911                                 xpt_release_ccb(start_ccb);
 912                                 ndaschedule(periph);
 913                                 return;
 914                         }
 915 #endif
 916                         KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
 917                             round_page(bp->bio_bcount + bp->bio_ma_offset) /
 918                             PAGE_SIZE == bp->bio_ma_n,
 919                             ("Short bio %p", bp));
 920                         nda_nvme_rw_bio(softc, &start_ccb->nvmeio, bp, bp->bio_cmd == BIO_READ ?
 921                             NVME_OPC_READ : NVME_OPC_WRITE);
 922                         break;
 923                 }
 924                 case BIO_DELETE:
 925                 {
 926                         struct nvme_dsm_range *dsm_range;
 927
 928                         dsm_range =
 929                             malloc(sizeof(*dsm_range), M_NVMEDA, M_ZERO | M_WAITOK);
 930                         dsm_range->length =
 931                             bp->bio_bcount / softc->disk->d_sectorsize;
 932                         dsm_range->starting_lba =
 933                             bp->bio_offset / softc->disk->d_sectorsize;
 934                         bp->bio_driver2 = dsm_range;
 935                         nda_nvme_trim(softc, &start_ccb->nvmeio, dsm_range, 1);
 936                         start_ccb->ccb_h.ccb_state = NDA_CCB_TRIM;
 937                         start_ccb->ccb_h.flags |= CAM_UNLOCKED;
 938                         cam_iosched_submit_trim(softc->cam_iosched);    /* XXX */
 939                         goto out;
 940                 }
 941                 case BIO_FLUSH:
 942                         nda_nvme_flush(softc, nvmeio);
 943                         break;
 944                 }
 945                 start_ccb->ccb_h.ccb_state = NDA_CCB_BUFFER_IO;
 946                 start_ccb->ccb_h.flags |= CAM_UNLOCKED;
 947 out:
 948                 start_ccb->ccb_h.ccb_bp = bp;
 949                 softc->outstanding_cmds++;
 950                 softc->refcount++;
 951                 cam_periph_unlock(periph);
 952                 xpt_action(start_ccb);
 953                 cam_periph_lock(periph);
 954                 softc->refcount--;
 955
 956                 /* May have more work to do, so ensure we stay scheduled */
 957                 ndaschedule(periph);
 958                 break;
 959                 }
 960         }
 961 }
 962
 963 static void
 964 ndadone(struct cam_periph *periph, union ccb *done_ccb)
 965 {
 966         struct nda_softc *softc;
 967         struct ccb_nvmeio *nvmeio = &done_ccb->nvmeio;
 968         struct cam_path *path;
 969         int state;
 970
 971         softc = (struct nda_softc *)periph->softc;
 972         path = done_ccb->ccb_h.path;
 973
 974         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("ndadone\n"));
 975
 976         state = nvmeio->ccb_h.ccb_state & NDA_CCB_TYPE_MASK;
 977         switch (state) {
 978         case NDA_CCB_BUFFER_IO:
 979         case NDA_CCB_TRIM:
 980         {
 981                 struct bio *bp;
 982                 int error;
 983
 984                 cam_periph_lock(periph);
 985                 bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
 986                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
 987                         error = ndaerror(done_ccb, 0, 0);
 988                         if (error == ERESTART) {
 989                                 /* A retry was scheduled, so just return. */
 990                                 cam_periph_unlock(periph);
 991                                 return;
 992                         }
 993                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
 994                                 cam_release_devq(path,
 995                                                  /*relsim_flags*/0,
 996                                                  /*reduction*/0,
 997                                                  /*timeout*/0,
 998                                                  /*getcount_only*/0);
 999                 } else {
1000                         if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
1001                                 panic("REQ_CMP with QFRZN");
1002                         error = 0;
1003                 }
1004                 bp->bio_error = error;
1005                 if (error != 0) {
1006                         bp->bio_resid = bp->bio_bcount;
1007                         bp->bio_flags |= BIO_ERROR;
1008                 } else {
1009                         if (state == NDA_CCB_TRIM)
1010                                 bp->bio_resid = 0;
1011                         else
1012                                 bp->bio_resid = nvmeio->resid;
1013                         if (bp->bio_resid > 0)
1014                                 bp->bio_flags |= BIO_ERROR;
1015                 }
1016                 if (state == NDA_CCB_TRIM)
1017                         free(bp->bio_driver2, M_NVMEDA);
1018                 softc->outstanding_cmds--;
1019
1020                 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb);
1021                 xpt_release_ccb(done_ccb);
1022                 if (state == NDA_CCB_TRIM) {
1023 #ifdef notyet
1024                         TAILQ_HEAD(, bio) queue;
1025                         struct bio *bp1;
1026
1027                         TAILQ_INIT(&queue);
1028                         TAILQ_CONCAT(&queue, &softc->trim_req.bps, bio_queue);
1029 #endif
1030                         cam_iosched_trim_done(softc->cam_iosched);
1031                         ndaschedule(periph);
1032                         cam_periph_unlock(periph);
1033 #ifdef notyet
1034 /* Not yet collapsing several BIO_DELETE requests into one TRIM */
1035                         while ((bp1 = TAILQ_FIRST(&queue)) != NULL) {
1036                                 TAILQ_REMOVE(&queue, bp1, bio_queue);
1037                                 bp1->bio_error = error;
1038                                 if (error != 0) {
1039                                         bp1->bio_flags |= BIO_ERROR;
1040                                         bp1->bio_resid = bp1->bio_bcount;
1041                                 } else
1042                                         bp1->bio_resid = 0;
1043                                 biodone(bp1);
1044                         }
1045 #else
1046                         biodone(bp);
1047 #endif
1048                 } else {
1049                         ndaschedule(periph);
1050                         cam_periph_unlock(periph);
1051                         biodone(bp);
1052                 }
1053                 return;
1054         }
1055         case NDA_CCB_DUMP:
1056                 /* No-op.  We're polling */
1057                 return;
1058         default:
1059                 break;
1060         }
1061         xpt_release_ccb(done_ccb);
1062 }
1063
1064 static int
1065 ndaerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
1066 {
1067         struct nda_softc *softc;
1068         struct cam_periph *periph;
1069
1070         periph = xpt_path_periph(ccb->ccb_h.path);
1071         softc = (struct nda_softc *)periph->softc;
1072
1073         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
1074         case CAM_CMD_TIMEOUT:
1075 #ifdef CAM_IO_STATS
1076                 softc->timeouts++;
1077 #endif
1078                 break;
1079         case CAM_REQ_ABORTED:
1080         case CAM_REQ_CMP_ERR:
1081         case CAM_REQ_TERMIO:
1082         case CAM_UNREC_HBA_ERROR:
1083         case CAM_DATA_RUN_ERR:
1084         case CAM_ATA_STATUS_ERROR:
1085 #ifdef CAM_IO_STATS
1086                 softc->errors++;
1087 #endif
1088                 break;
1089         default:
1090                 break;
1091         }
1092
1093         return(cam_periph_error(ccb, cam_flags, sense_flags, NULL));
1094 }
1095
1096 /*
1097  * Step through all NDA peripheral drivers, and if the device is still open,
1098  * sync the disk cache to physical media.
1099  */
1100 static void
1101 ndaflush(void)
1102 {
1103         struct cam_periph *periph;
1104         struct nda_softc *softc;
1105         union ccb *ccb;
1106         int error;
1107
1108         CAM_PERIPH_FOREACH(periph, &ndadriver) {
1109                 softc = (struct nda_softc *)periph->softc;
1110                 if (SCHEDULER_STOPPED()) {
1111                         /* If we paniced with the lock held, do not recurse. */
1112                         if (!cam_periph_owned(periph) &&
1113                             (softc->flags & NDA_FLAG_OPEN)) {
1114                                 ndadump(softc->disk, NULL, 0, 0, 0);
1115                         }
1116                         continue;
1117                 }
1118                 cam_periph_lock(periph);
1119                 /*
1120                  * We only sync the cache if the drive is still open, and
1121                  * if the drive is capable of it..
1122                  */
1123                 if ((softc->flags & NDA_FLAG_OPEN) == 0) {
1124                         cam_periph_unlock(periph);
1125                         continue;
1126                 }
1127
1128                 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
1129                 nda_nvme_flush(softc, &ccb->nvmeio);
1130                 error = cam_periph_runccb(ccb, ndaerror, /*cam_flags*/0,
1131                     /*sense_flags*/ SF_NO_RECOVERY | SF_NO_RETRY,
1132                     softc->disk->d_devstat);
1133                 if (error != 0)
1134                         xpt_print(periph->path, "Synchronize cache failed\n");
1135                 xpt_release_ccb(ccb);
1136                 cam_periph_unlock(periph);
1137         }
1138 }
1139
1140 static void
1141 ndashutdown(void *arg, int howto)
1142 {
1143
1144         ndaflush();
1145 }
1146
1147 static void
1148 ndasuspend(void *arg)
1149 {
1150
1151         ndaflush();
1152 }