sys/cam/ctl/ctl_backend_block.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2003 Silicon Graphics International Corp.
   5  * Copyright (c) 2009-2011 Spectra Logic Corporation
   6  * Copyright (c) 2012,2021 The FreeBSD Foundation
   7  * Copyright (c) 2014-2021 Alexander Motin <mav@FreeBSD.org>
   8  * All rights reserved.
   9  *
  10  * Portions of this software were developed by Edward Tomasz Napierala
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Portions of this software were developed by Ka Ho Ng <khng@FreeBSD.org>
  14  * under sponsorship from the FreeBSD Foundation.
  15  *
  16  * Redistribution and use in source and binary forms, with or without
  17  * modification, are permitted provided that the following conditions
  18  * are met:
  19  * 1. Redistributions of source code must retain the above copyright
  20  *    notice, this list of conditions, and the following disclaimer,
  21  *    without modification.
  22  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  23  *    substantially similar to the "NO WARRANTY" disclaimer below
  24  *    ("Disclaimer") and any redistribution must be conditioned upon
  25  *    including a substantially similar Disclaimer requirement for further
  26  *    binary redistribution.
  27  *
  28  * NO WARRANTY
  29  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  30  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  31  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  32  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  33  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  37  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  38  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  39  * POSSIBILITY OF SUCH DAMAGES.
  40  *
  41  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
  42  */
  43 /*
  44  * CAM Target Layer driver backend for block devices.
  45  *
  46  * Author: Ken Merry <ken@FreeBSD.org>
  47  */
  48 #include <sys/cdefs.h>
  49 __FBSDID("$FreeBSD$");
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/kernel.h>
  54 #include <sys/types.h>
  55 #include <sys/kthread.h>
  56 #include <sys/bio.h>
  57 #include <sys/fcntl.h>
  58 #include <sys/limits.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/condvar.h>
  62 #include <sys/malloc.h>
  63 #include <sys/conf.h>
  64 #include <sys/ioccom.h>
  65 #include <sys/queue.h>
  66 #include <sys/sbuf.h>
  67 #include <sys/endian.h>
  68 #include <sys/uio.h>
  69 #include <sys/buf.h>
  70 #include <sys/taskqueue.h>
  71 #include <sys/vnode.h>
  72 #include <sys/namei.h>
  73 #include <sys/mount.h>
  74 #include <sys/disk.h>
  75 #include <sys/fcntl.h>
  76 #include <sys/filedesc.h>
  77 #include <sys/filio.h>
  78 #include <sys/proc.h>
  79 #include <sys/pcpu.h>
  80 #include <sys/module.h>
  81 #include <sys/sdt.h>
  82 #include <sys/devicestat.h>
  83 #include <sys/sysctl.h>
  84 #include <sys/nv.h>
  85 #include <sys/dnv.h>
  86 #include <sys/sx.h>
  87 #include <sys/unistd.h>
  88
  89 #include <geom/geom.h>
  90
  91 #include <cam/cam.h>
  92 #include <cam/scsi/scsi_all.h>
  93 #include <cam/scsi/scsi_da.h>
  94 #include <cam/ctl/ctl_io.h>
  95 #include <cam/ctl/ctl.h>
  96 #include <cam/ctl/ctl_backend.h>
  97 #include <cam/ctl/ctl_ioctl.h>
  98 #include <cam/ctl/ctl_ha.h>
  99 #include <cam/ctl/ctl_scsi_all.h>
 100 #include <cam/ctl/ctl_private.h>
 101 #include <cam/ctl/ctl_error.h>
 102
 103 /*
 104  * The idea here is to allocate enough S/G space to handle at least 1MB I/Os.
 105  * On systems with small maxphys it can be 8 128KB segments.  On large systems
 106  * it can be up to 8 1MB segments.  I/Os larger than that we'll split.
 107  */
 108 #define CTLBLK_MAX_SEGS         8
 109 #define CTLBLK_HALF_SEGS        (CTLBLK_MAX_SEGS / 2)
 110 #define CTLBLK_MIN_SEG          (128 * 1024)
 111 #define CTLBLK_MAX_SEG          MIN(1024 * 1024, MAX(CTLBLK_MIN_SEG, maxphys))
 112 #define CTLBLK_MAX_IO_SIZE      (CTLBLK_MAX_SEG * CTLBLK_MAX_SEGS)
 113
 114 #ifdef CTLBLK_DEBUG
 115 #define DPRINTF(fmt, args...) \
 116     printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args)
 117 #else
 118 #define DPRINTF(fmt, args...) do {} while(0)
 119 #endif
 120
 121 #define PRIV(io)        \
 122     ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND])
 123 #define ARGS(io)        \
 124     ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN])
 125
 126 SDT_PROVIDER_DEFINE(cbb);
 127
 128 typedef enum {
 129         CTL_BE_BLOCK_LUN_UNCONFIGURED   = 0x01,
 130         CTL_BE_BLOCK_LUN_WAITING        = 0x04,
 131 } ctl_be_block_lun_flags;
 132
 133 typedef enum {
 134         CTL_BE_BLOCK_NONE,
 135         CTL_BE_BLOCK_DEV,
 136         CTL_BE_BLOCK_FILE
 137 } ctl_be_block_type;
 138
 139 struct ctl_be_block_filedata {
 140         struct ucred *cred;
 141 };
 142
 143 union ctl_be_block_bedata {
 144         struct ctl_be_block_filedata file;
 145 };
 146
 147 struct ctl_be_block_io;
 148 struct ctl_be_block_lun;
 149
 150 typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
 151                                struct ctl_be_block_io *beio);
 152 typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun,
 153                                   const char *attrname);
 154
 155 /*
 156  * Backend LUN structure.  There is a 1:1 mapping between a block device
 157  * and a backend block LUN, and between a backend block LUN and a CTL LUN.
 158  */
 159 struct ctl_be_block_lun {
 160         struct ctl_be_lun cbe_lun;              /* Must be first element. */
 161         struct ctl_lun_create_params params;
 162         char *dev_path;
 163         ctl_be_block_type dev_type;
 164         struct vnode *vn;
 165         union ctl_be_block_bedata backend;
 166         cbb_dispatch_t dispatch;
 167         cbb_dispatch_t lun_flush;
 168         cbb_dispatch_t unmap;
 169         cbb_dispatch_t get_lba_status;
 170         cbb_getattr_t getattr;
 171         uint64_t size_blocks;
 172         uint64_t size_bytes;
 173         struct ctl_be_block_softc *softc;
 174         struct devstat *disk_stats;
 175         ctl_be_block_lun_flags flags;
 176         SLIST_ENTRY(ctl_be_block_lun) links;
 177         struct taskqueue *io_taskqueue;
 178         struct task io_task;
 179         int num_threads;
 180         STAILQ_HEAD(, ctl_io_hdr) input_queue;
 181         STAILQ_HEAD(, ctl_io_hdr) config_read_queue;
 182         STAILQ_HEAD(, ctl_io_hdr) config_write_queue;
 183         STAILQ_HEAD(, ctl_io_hdr) datamove_queue;
 184         struct mtx_padalign io_lock;
 185         struct mtx_padalign queue_lock;
 186 };
 187
 188 /*
 189  * Overall softc structure for the block backend module.
 190  */
 191 struct ctl_be_block_softc {
 192         struct sx                        modify_lock;
 193         struct mtx                       lock;
 194         int                              num_luns;
 195         SLIST_HEAD(, ctl_be_block_lun)   lun_list;
 196         uma_zone_t                       beio_zone;
 197         uma_zone_t                       bufmin_zone;
 198         uma_zone_t                       bufmax_zone;
 199 };
 200
 201 static struct ctl_be_block_softc backend_block_softc;
 202
 203 /*
 204  * Per-I/O information.
 205  */
 206 struct ctl_be_block_io {
 207         union ctl_io                    *io;
 208         struct ctl_sg_entry             sg_segs[CTLBLK_MAX_SEGS];
 209         struct iovec                    xiovecs[CTLBLK_MAX_SEGS];
 210         int                             refcnt;
 211         int                             bio_cmd;
 212         int                             two_sglists;
 213         int                             num_segs;
 214         int                             num_bios_sent;
 215         int                             num_bios_done;
 216         int                             send_complete;
 217         int                             first_error;
 218         uint64_t                        first_error_offset;
 219         struct bintime                  ds_t0;
 220         devstat_tag_type                ds_tag_type;
 221         devstat_trans_flags             ds_trans_type;
 222         uint64_t                        io_len;
 223         uint64_t                        io_offset;
 224         int                             io_arg;
 225         struct ctl_be_block_softc       *softc;
 226         struct ctl_be_block_lun         *lun;
 227         void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */
 228 };
 229
 230 extern struct ctl_softc *control_softc;
 231
 232 static int cbb_num_threads = 32;
 233 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 234             "CAM Target Layer Block Backend");
 235 SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN,
 236            &cbb_num_threads, 0, "Number of threads per backing file");
 237
 238 static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
 239 static void ctl_free_beio(struct ctl_be_block_io *beio);
 240 static void ctl_complete_beio(struct ctl_be_block_io *beio);
 241 static int ctl_be_block_move_done(union ctl_io *io, bool samethr);
 242 static void ctl_be_block_biodone(struct bio *bio);
 243 static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 244                                     struct ctl_be_block_io *beio);
 245 static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 246                                        struct ctl_be_block_io *beio);
 247 static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
 248                                   struct ctl_be_block_io *beio);
 249 static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun,
 250                                          const char *attrname);
 251 static void ctl_be_block_unmap_file(struct ctl_be_block_lun *be_lun,
 252                                     struct ctl_be_block_io *beio);
 253 static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 254                                    struct ctl_be_block_io *beio);
 255 static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
 256                                    struct ctl_be_block_io *beio);
 257 static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
 258                                       struct ctl_be_block_io *beio);
 259 static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun,
 260                                          const char *attrname);
 261 static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
 262                                     union ctl_io *io);
 263 static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
 264                                     union ctl_io *io);
 265 static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 266                                   union ctl_io *io);
 267 static void ctl_be_block_worker(void *context, int pending);
 268 static int ctl_be_block_submit(union ctl_io *io);
 269 static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
 270                                    int flag, struct thread *td);
 271 static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
 272                                   struct ctl_lun_req *req);
 273 static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
 274                                  struct ctl_lun_req *req);
 275 static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
 276 static int ctl_be_block_open(struct ctl_be_block_lun *be_lun,
 277                              struct ctl_lun_req *req);
 278 static int ctl_be_block_create(struct ctl_be_block_softc *softc,
 279                                struct ctl_lun_req *req);
 280 static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
 281                            struct ctl_lun_req *req);
 282 static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
 283                            struct ctl_lun_req *req);
 284 static void ctl_be_block_lun_shutdown(struct ctl_be_lun *cbe_lun);
 285 static int ctl_be_block_config_write(union ctl_io *io);
 286 static int ctl_be_block_config_read(union ctl_io *io);
 287 static int ctl_be_block_lun_info(struct ctl_be_lun *cbe_lun, struct sbuf *sb);
 288 static uint64_t ctl_be_block_lun_attr(struct ctl_be_lun *cbe_lun, const char *attrname);
 289 static int ctl_be_block_init(void);
 290 static int ctl_be_block_shutdown(void);
 291
 292 static struct ctl_backend_driver ctl_be_block_driver =
 293 {
 294         .name = "block",
 295         .flags = CTL_BE_FLAG_HAS_CONFIG,
 296         .init = ctl_be_block_init,
 297         .shutdown = ctl_be_block_shutdown,
 298         .data_submit = ctl_be_block_submit,
 299         .config_read = ctl_be_block_config_read,
 300         .config_write = ctl_be_block_config_write,
 301         .ioctl = ctl_be_block_ioctl,
 302         .lun_info = ctl_be_block_lun_info,
 303         .lun_attr = ctl_be_block_lun_attr
 304 };
 305
 306 MALLOC_DEFINE(M_CTLBLK, "ctlblock", "Memory used for CTL block backend");
 307 CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
 308
 309 static void
 310 ctl_alloc_seg(struct ctl_be_block_softc *softc, struct ctl_sg_entry *sg,
 311     size_t len)
 312 {
 313
 314         if (len <= CTLBLK_MIN_SEG) {
 315                 sg->addr = uma_zalloc(softc->bufmin_zone, M_WAITOK);
 316         } else {
 317                 KASSERT(len <= CTLBLK_MAX_SEG,
 318                     ("Too large alloc %zu > %lu", len, CTLBLK_MAX_SEG));
 319                 sg->addr = uma_zalloc(softc->bufmax_zone, M_WAITOK);
 320         }
 321         sg->len = len;
 322 }
 323
 324 static void
 325 ctl_free_seg(struct ctl_be_block_softc *softc, struct ctl_sg_entry *sg)
 326 {
 327
 328         if (sg->len <= CTLBLK_MIN_SEG) {
 329                 uma_zfree(softc->bufmin_zone, sg->addr);
 330         } else {
 331                 KASSERT(sg->len <= CTLBLK_MAX_SEG,
 332                     ("Too large free %zu > %lu", sg->len, CTLBLK_MAX_SEG));
 333                 uma_zfree(softc->bufmax_zone, sg->addr);
 334         }
 335 }
 336
 337 static struct ctl_be_block_io *
 338 ctl_alloc_beio(struct ctl_be_block_softc *softc)
 339 {
 340         struct ctl_be_block_io *beio;
 341
 342         beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO);
 343         beio->softc = softc;
 344         beio->refcnt = 1;
 345         return (beio);
 346 }
 347
 348 static void
 349 ctl_real_free_beio(struct ctl_be_block_io *beio)
 350 {
 351         struct ctl_be_block_softc *softc = beio->softc;
 352         int i;
 353
 354         for (i = 0; i < beio->num_segs; i++) {
 355                 ctl_free_seg(softc, &beio->sg_segs[i]);
 356
 357                 /* For compare we had two equal S/G lists. */
 358                 if (beio->two_sglists) {
 359                         ctl_free_seg(softc,
 360                             &beio->sg_segs[i + CTLBLK_HALF_SEGS]);
 361                 }
 362         }
 363
 364         uma_zfree(softc->beio_zone, beio);
 365 }
 366
 367 static void
 368 ctl_refcnt_beio(void *arg, int diff)
 369 {
 370         struct ctl_be_block_io *beio = arg;
 371
 372         if (atomic_fetchadd_int(&beio->refcnt, diff) + diff == 0)
 373                 ctl_real_free_beio(beio);
 374 }
 375
 376 static void
 377 ctl_free_beio(struct ctl_be_block_io *beio)
 378 {
 379
 380         ctl_refcnt_beio(beio, -1);
 381 }
 382
 383 static void
 384 ctl_complete_beio(struct ctl_be_block_io *beio)
 385 {
 386         union ctl_io *io = beio->io;
 387
 388         if (beio->beio_cont != NULL) {
 389                 beio->beio_cont(beio);
 390         } else {
 391                 ctl_free_beio(beio);
 392                 ctl_data_submit_done(io);
 393         }
 394 }
 395
 396 static size_t
 397 cmp(uint8_t *a, uint8_t *b, size_t size)
 398 {
 399         size_t i;
 400
 401         for (i = 0; i < size; i++) {
 402                 if (a[i] != b[i])
 403                         break;
 404         }
 405         return (i);
 406 }
 407
 408 static void
 409 ctl_be_block_compare(union ctl_io *io)
 410 {
 411         struct ctl_be_block_io *beio;
 412         uint64_t off, res;
 413         int i;
 414         uint8_t info[8];
 415
 416         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 417         off = 0;
 418         for (i = 0; i < beio->num_segs; i++) {
 419                 res = cmp(beio->sg_segs[i].addr,
 420                     beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
 421                     beio->sg_segs[i].len);
 422                 off += res;
 423                 if (res < beio->sg_segs[i].len)
 424                         break;
 425         }
 426         if (i < beio->num_segs) {
 427                 scsi_u64to8b(off, info);
 428                 ctl_set_sense(&io->scsiio, /*current_error*/ 1,
 429                     /*sense_key*/ SSD_KEY_MISCOMPARE,
 430                     /*asc*/ 0x1D, /*ascq*/ 0x00,
 431                     /*type*/ SSD_ELEM_INFO,
 432                     /*size*/ sizeof(info), /*data*/ &info,
 433                     /*type*/ SSD_ELEM_NONE);
 434         } else
 435                 ctl_set_success(&io->scsiio);
 436 }
 437
 438 static int
 439 ctl_be_block_move_done(union ctl_io *io, bool samethr)
 440 {
 441         struct ctl_be_block_io *beio;
 442         struct ctl_be_block_lun *be_lun;
 443         struct ctl_lba_len_flags *lbalen;
 444
 445         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 446
 447         DPRINTF("entered\n");
 448         io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 449
 450         /*
 451          * We set status at this point for read and compare commands.
 452          */
 453         if ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0 &&
 454             (io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE) {
 455                 lbalen = ARGS(io);
 456                 if (lbalen->flags & CTL_LLF_READ) {
 457                         ctl_set_success(&io->scsiio);
 458                 } else if (lbalen->flags & CTL_LLF_COMPARE) {
 459                         /* We have two data blocks ready for comparison. */
 460                         ctl_be_block_compare(io);
 461                 }
 462         }
 463
 464         /*
 465          * If this is a read, or a write with errors, it is done.
 466          */
 467         if ((beio->bio_cmd == BIO_READ)
 468          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)
 469          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) {
 470                 ctl_complete_beio(beio);
 471                 return (0);
 472         }
 473
 474         /*
 475          * At this point, we have a write and the DMA completed successfully.
 476          * If we were called synchronously in the original thread then just
 477          * dispatch, otherwise we now have to queue it to the task queue to
 478          * execute the backend I/O.  That is because we do blocking
 479          * memory allocations, and in the file backing case, blocking I/O.
 480          * This move done routine is generally called in the SIM's
 481          * interrupt context, and therefore we cannot block.
 482          */
 483         be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);
 484         if (samethr) {
 485                 be_lun->dispatch(be_lun, beio);
 486         } else {
 487                 mtx_lock(&be_lun->queue_lock);
 488                 STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
 489                 mtx_unlock(&be_lun->queue_lock);
 490                 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
 491         }
 492         return (0);
 493 }
 494
 495 static void
 496 ctl_be_block_biodone(struct bio *bio)
 497 {
 498         struct ctl_be_block_io *beio = bio->bio_caller1;
 499         struct ctl_be_block_lun *be_lun = beio->lun;
 500         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
 501         union ctl_io *io;
 502         int error;
 503
 504         io = beio->io;
 505
 506         DPRINTF("entered\n");
 507
 508         error = bio->bio_error;
 509         mtx_lock(&be_lun->io_lock);
 510         if (error != 0 &&
 511             (beio->first_error == 0 ||
 512              bio->bio_offset < beio->first_error_offset)) {
 513                 beio->first_error = error;
 514                 beio->first_error_offset = bio->bio_offset;
 515         }
 516
 517         beio->num_bios_done++;
 518
 519         /*
 520          * XXX KDM will this cause WITNESS to complain?  Holding a lock
 521          * during the free might cause it to complain.
 522          */
 523         g_destroy_bio(bio);
 524
 525         /*
 526          * If the send complete bit isn't set, or we aren't the last I/O to
 527          * complete, then we're done.
 528          */
 529         if ((beio->send_complete == 0)
 530          || (beio->num_bios_done < beio->num_bios_sent)) {
 531                 mtx_unlock(&be_lun->io_lock);
 532                 return;
 533         }
 534
 535         /*
 536          * At this point, we've verified that we are the last I/O to
 537          * complete, so it's safe to drop the lock.
 538          */
 539         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 540             beio->ds_tag_type, beio->ds_trans_type,
 541             /*now*/ NULL, /*then*/&beio->ds_t0);
 542         mtx_unlock(&be_lun->io_lock);
 543
 544         /*
 545          * If there are any errors from the backing device, we fail the
 546          * entire I/O with a medium error.
 547          */
 548         error = beio->first_error;
 549         if (error != 0) {
 550                 if (error == EOPNOTSUPP) {
 551                         ctl_set_invalid_opcode(&io->scsiio);
 552                 } else if (error == ENOSPC || error == EDQUOT) {
 553                         ctl_set_space_alloc_fail(&io->scsiio);
 554                 } else if (error == EROFS || error == EACCES) {
 555                         ctl_set_hw_write_protected(&io->scsiio);
 556                 } else if (beio->bio_cmd == BIO_FLUSH) {
 557                         /* XXX KDM is there is a better error here? */
 558                         ctl_set_internal_failure(&io->scsiio,
 559                                                  /*sks_valid*/ 1,
 560                                                  /*retry_count*/ 0xbad2);
 561                 } else {
 562                         ctl_set_medium_error(&io->scsiio,
 563                             beio->bio_cmd == BIO_READ);
 564                 }
 565                 ctl_complete_beio(beio);
 566                 return;
 567         }
 568
 569         /*
 570          * If this is a write, a flush, a delete or verify, we're all done.
 571          * If this is a read, we can now send the data to the user.
 572          */
 573         if ((beio->bio_cmd == BIO_WRITE)
 574          || (beio->bio_cmd == BIO_FLUSH)
 575          || (beio->bio_cmd == BIO_DELETE)
 576          || (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 577                 ctl_set_success(&io->scsiio);
 578                 ctl_complete_beio(beio);
 579         } else {
 580                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 581                     beio->beio_cont == NULL) {
 582                         ctl_set_success(&io->scsiio);
 583                         if (cbe_lun->serseq >= CTL_LUN_SERSEQ_SOFT)
 584                                 ctl_serseq_done(io);
 585                 }
 586                 ctl_datamove(io);
 587         }
 588 }
 589
 590 static void
 591 ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 592                         struct ctl_be_block_io *beio)
 593 {
 594         union ctl_io *io = beio->io;
 595         struct mount *mountpoint;
 596         int error;
 597
 598         DPRINTF("entered\n");
 599
 600         binuptime(&beio->ds_t0);
 601         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 602
 603         (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 604
 605         vn_lock(be_lun->vn, vn_lktype_write(mountpoint, be_lun->vn) |
 606             LK_RETRY);
 607         error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT,
 608             curthread);
 609         VOP_UNLOCK(be_lun->vn);
 610
 611         vn_finished_write(mountpoint);
 612
 613         mtx_lock(&be_lun->io_lock);
 614         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 615             beio->ds_tag_type, beio->ds_trans_type,
 616             /*now*/ NULL, /*then*/&beio->ds_t0);
 617         mtx_unlock(&be_lun->io_lock);
 618
 619         if (error == 0)
 620                 ctl_set_success(&io->scsiio);
 621         else {
 622                 /* XXX KDM is there is a better error here? */
 623                 ctl_set_internal_failure(&io->scsiio,
 624                                          /*sks_valid*/ 1,
 625                                          /*retry_count*/ 0xbad1);
 626         }
 627
 628         ctl_complete_beio(beio);
 629 }
 630
 631 SDT_PROBE_DEFINE1(cbb, , read, file_start, "uint64_t");
 632 SDT_PROBE_DEFINE1(cbb, , write, file_start, "uint64_t");
 633 SDT_PROBE_DEFINE1(cbb, , read, file_done,"uint64_t");
 634 SDT_PROBE_DEFINE1(cbb, , write, file_done, "uint64_t");
 635
 636 static void
 637 ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 638                            struct ctl_be_block_io *beio)
 639 {
 640         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
 641         struct ctl_be_block_filedata *file_data;
 642         union ctl_io *io;
 643         struct uio xuio;
 644         struct iovec *xiovec;
 645         size_t s;
 646         int error, flags, i;
 647
 648         DPRINTF("entered\n");
 649
 650         file_data = &be_lun->backend.file;
 651         io = beio->io;
 652         flags = 0;
 653         if (ARGS(io)->flags & CTL_LLF_DPO)
 654                 flags |= IO_DIRECT;
 655         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 656                 flags |= IO_SYNC;
 657
 658         bzero(&xuio, sizeof(xuio));
 659         if (beio->bio_cmd == BIO_READ) {
 660                 SDT_PROBE0(cbb, , read, file_start);
 661                 xuio.uio_rw = UIO_READ;
 662         } else {
 663                 SDT_PROBE0(cbb, , write, file_start);
 664                 xuio.uio_rw = UIO_WRITE;
 665         }
 666         xuio.uio_offset = beio->io_offset;
 667         xuio.uio_resid = beio->io_len;
 668         xuio.uio_segflg = UIO_SYSSPACE;
 669         xuio.uio_iov = beio->xiovecs;
 670         xuio.uio_iovcnt = beio->num_segs;
 671         xuio.uio_td = curthread;
 672
 673         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 674                 xiovec->iov_base = beio->sg_segs[i].addr;
 675                 xiovec->iov_len = beio->sg_segs[i].len;
 676         }
 677
 678         binuptime(&beio->ds_t0);
 679         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 680
 681         if (beio->bio_cmd == BIO_READ) {
 682                 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 683
 684                 if (beio->beio_cont == NULL &&
 685                     cbe_lun->serseq == CTL_LUN_SERSEQ_SOFT)
 686                         ctl_serseq_done(io);
 687                 /*
 688                  * UFS pays attention to IO_DIRECT for reads.  If the
 689                  * DIRECTIO option is configured into the kernel, it calls
 690                  * ffs_rawread().  But that only works for single-segment
 691                  * uios with user space addresses.  In our case, with a
 692                  * kernel uio, it still reads into the buffer cache, but it
 693                  * will just try to release the buffer from the cache later
 694                  * on in ffs_read().
 695                  *
 696                  * ZFS does not pay attention to IO_DIRECT for reads.
 697                  *
 698                  * UFS does not pay attention to IO_SYNC for reads.
 699                  *
 700                  * ZFS pays attention to IO_SYNC (which translates into the
 701                  * Solaris define FRSYNC for zfs_read()) for reads.  It
 702                  * attempts to sync the file before reading.
 703                  */
 704                 error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred);
 705
 706                 VOP_UNLOCK(be_lun->vn);
 707                 SDT_PROBE0(cbb, , read, file_done);
 708                 if (error == 0 && xuio.uio_resid > 0) {
 709                         /*
 710                          * If we red less then requested (EOF), then
 711                          * we should clean the rest of the buffer.
 712                          */
 713                         s = beio->io_len - xuio.uio_resid;
 714                         for (i = 0; i < beio->num_segs; i++) {
 715                                 if (s >= beio->sg_segs[i].len) {
 716                                         s -= beio->sg_segs[i].len;
 717                                         continue;
 718                                 }
 719                                 bzero((uint8_t *)beio->sg_segs[i].addr + s,
 720                                     beio->sg_segs[i].len - s);
 721                                 s = 0;
 722                         }
 723                 }
 724         } else {
 725                 struct mount *mountpoint;
 726
 727                 (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 728                 vn_lock(be_lun->vn, vn_lktype_write(mountpoint,
 729                     be_lun->vn) | LK_RETRY);
 730
 731                 /*
 732                  * UFS pays attention to IO_DIRECT for writes.  The write
 733                  * is done asynchronously.  (Normally the write would just
 734                  * get put into cache.
 735                  *
 736                  * UFS pays attention to IO_SYNC for writes.  It will
 737                  * attempt to write the buffer out synchronously if that
 738                  * flag is set.
 739                  *
 740                  * ZFS does not pay attention to IO_DIRECT for writes.
 741                  *
 742                  * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC)
 743                  * for writes.  It will flush the transaction from the
 744                  * cache before returning.
 745                  */
 746                 error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred);
 747                 VOP_UNLOCK(be_lun->vn);
 748
 749                 vn_finished_write(mountpoint);
 750                 SDT_PROBE0(cbb, , write, file_done);
 751         }
 752
 753         mtx_lock(&be_lun->io_lock);
 754         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 755             beio->ds_tag_type, beio->ds_trans_type,
 756             /*now*/ NULL, /*then*/&beio->ds_t0);
 757         mtx_unlock(&be_lun->io_lock);
 758
 759         /*
 760          * If we got an error, set the sense data to "MEDIUM ERROR" and
 761          * return the I/O to the user.
 762          */
 763         if (error != 0) {
 764                 if (error == ENOSPC || error == EDQUOT) {
 765                         ctl_set_space_alloc_fail(&io->scsiio);
 766                 } else if (error == EROFS || error == EACCES) {
 767                         ctl_set_hw_write_protected(&io->scsiio);
 768                 } else {
 769                         ctl_set_medium_error(&io->scsiio,
 770                             beio->bio_cmd == BIO_READ);
 771                 }
 772                 ctl_complete_beio(beio);
 773                 return;
 774         }
 775
 776         /*
 777          * If this is a write or a verify, we're all done.
 778          * If this is a read, we can now send the data to the user.
 779          */
 780         if ((beio->bio_cmd == BIO_WRITE) ||
 781             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 782                 ctl_set_success(&io->scsiio);
 783                 ctl_complete_beio(beio);
 784         } else {
 785                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 786                     beio->beio_cont == NULL) {
 787                         ctl_set_success(&io->scsiio);
 788                         if (cbe_lun->serseq > CTL_LUN_SERSEQ_SOFT)
 789                                 ctl_serseq_done(io);
 790                 }
 791                 ctl_datamove(io);
 792         }
 793 }
 794
 795 static void
 796 ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
 797                         struct ctl_be_block_io *beio)
 798 {
 799         union ctl_io *io = beio->io;
 800         struct ctl_lba_len_flags *lbalen = ARGS(io);
 801         struct scsi_get_lba_status_data *data;
 802         off_t roff, off;
 803         int error, status;
 804
 805         DPRINTF("entered\n");
 806
 807         off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
 808         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 809         error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off,
 810             0, curthread->td_ucred, curthread);
 811         if (error == 0 && off > roff)
 812                 status = 0;     /* mapped up to off */
 813         else {
 814                 error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off,
 815                     0, curthread->td_ucred, curthread);
 816                 if (error == 0 && off > roff)
 817                         status = 1;     /* deallocated up to off */
 818                 else {
 819                         status = 0;     /* unknown up to the end */
 820                         off = be_lun->size_bytes;
 821                 }
 822         }
 823         VOP_UNLOCK(be_lun->vn);
 824
 825         data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
 826         scsi_u64to8b(lbalen->lba, data->descr[0].addr);
 827         scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
 828             lbalen->lba), data->descr[0].length);
 829         data->descr[0].status = status;
 830
 831         ctl_complete_beio(beio);
 832 }
 833
 834 static uint64_t
 835 ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname)
 836 {
 837         struct vattr            vattr;
 838         struct statfs           statfs;
 839         uint64_t                val;
 840         int                     error;
 841
 842         val = UINT64_MAX;
 843         if (be_lun->vn == NULL)
 844                 return (val);
 845         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 846         if (strcmp(attrname, "blocksused") == 0) {
 847                 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
 848                 if (error == 0)
 849                         val = vattr.va_bytes / be_lun->cbe_lun.blocksize;
 850         }
 851         if (strcmp(attrname, "blocksavail") == 0 &&
 852             !VN_IS_DOOMED(be_lun->vn)) {
 853                 error = VFS_STATFS(be_lun->vn->v_mount, &statfs);
 854                 if (error == 0)
 855                         val = statfs.f_bavail * statfs.f_bsize /
 856                             be_lun->cbe_lun.blocksize;
 857         }
 858         VOP_UNLOCK(be_lun->vn);
 859         return (val);
 860 }
 861
 862 static void
 863 ctl_be_block_unmap_file(struct ctl_be_block_lun *be_lun,
 864                         struct ctl_be_block_io *beio)
 865 {
 866         struct ctl_be_block_filedata *file_data;
 867         union ctl_io *io;
 868         struct ctl_ptr_len_flags *ptrlen;
 869         struct scsi_unmap_desc *buf, *end;
 870         struct mount *mp;
 871         off_t off, len;
 872         int error;
 873
 874         io = beio->io;
 875         file_data = &be_lun->backend.file;
 876         mp = NULL;
 877         error = 0;
 878
 879         binuptime(&beio->ds_t0);
 880         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
 881
 882         (void)vn_start_write(be_lun->vn, &mp, V_WAIT);
 883         vn_lock(be_lun->vn, vn_lktype_write(mp, be_lun->vn) | LK_RETRY);
 884         if (beio->io_offset == -1) {
 885                 beio->io_len = 0;
 886                 ptrlen = (struct ctl_ptr_len_flags *)
 887                     &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
 888                 buf = (struct scsi_unmap_desc *)ptrlen->ptr;
 889                 end = buf + ptrlen->len / sizeof(*buf);
 890                 for (; buf < end; buf++) {
 891                         off = (off_t)scsi_8btou64(buf->lba) *
 892                             be_lun->cbe_lun.blocksize;
 893                         len = (off_t)scsi_4btoul(buf->length) *
 894                             be_lun->cbe_lun.blocksize;
 895                         beio->io_len += len;
 896                         error = vn_deallocate(be_lun->vn, &off, &len,
 897                             0, IO_NOMACCHECK | IO_NODELOCKED, file_data->cred,
 898                             NOCRED);
 899                         if (error != 0)
 900                                 break;
 901                 }
 902         } else {
 903                 /* WRITE_SAME */
 904                 off = beio->io_offset;
 905                 len = beio->io_len;
 906                 error = vn_deallocate(be_lun->vn, &off, &len, 0,
 907                     IO_NOMACCHECK | IO_NODELOCKED, file_data->cred, NOCRED);
 908         }
 909         VOP_UNLOCK(be_lun->vn);
 910         vn_finished_write(mp);
 911
 912         mtx_lock(&be_lun->io_lock);
 913         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 914             beio->ds_tag_type, beio->ds_trans_type,
 915             /*now*/ NULL, /*then*/&beio->ds_t0);
 916         mtx_unlock(&be_lun->io_lock);
 917
 918         /*
 919          * If we got an error, set the sense data to "MEDIUM ERROR" and
 920          * return the I/O to the user.
 921          */
 922         switch (error) {
 923         case 0:
 924                 ctl_set_success(&io->scsiio);
 925                 break;
 926         case ENOSPC:
 927         case EDQUOT:
 928                 ctl_set_space_alloc_fail(&io->scsiio);
 929                 break;
 930         case EROFS:
 931         case EACCES:
 932                 ctl_set_hw_write_protected(&io->scsiio);
 933                 break;
 934         default:
 935                 ctl_set_medium_error(&io->scsiio, false);
 936         }
 937         ctl_complete_beio(beio);
 938 }
 939
 940 static void
 941 ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
 942                            struct ctl_be_block_io *beio)
 943 {
 944         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
 945         union ctl_io *io;
 946         struct cdevsw *csw;
 947         struct cdev *dev;
 948         struct uio xuio;
 949         struct iovec *xiovec;
 950         int error, flags, i, ref;
 951
 952         DPRINTF("entered\n");
 953
 954         io = beio->io;
 955         flags = 0;
 956         if (ARGS(io)->flags & CTL_LLF_DPO)
 957                 flags |= IO_DIRECT;
 958         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 959                 flags |= IO_SYNC;
 960
 961         bzero(&xuio, sizeof(xuio));
 962         if (beio->bio_cmd == BIO_READ) {
 963                 SDT_PROBE0(cbb, , read, file_start);
 964                 xuio.uio_rw = UIO_READ;
 965         } else {
 966                 SDT_PROBE0(cbb, , write, file_start);
 967                 xuio.uio_rw = UIO_WRITE;
 968         }
 969         xuio.uio_offset = beio->io_offset;
 970         xuio.uio_resid = beio->io_len;
 971         xuio.uio_segflg = UIO_SYSSPACE;
 972         xuio.uio_iov = beio->xiovecs;
 973         xuio.uio_iovcnt = beio->num_segs;
 974         xuio.uio_td = curthread;
 975
 976         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 977                 xiovec->iov_base = beio->sg_segs[i].addr;
 978                 xiovec->iov_len = beio->sg_segs[i].len;
 979         }
 980
 981         binuptime(&beio->ds_t0);
 982         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 983
 984         csw = devvn_refthread(be_lun->vn, &dev, &ref);
 985         if (csw) {
 986                 if (beio->bio_cmd == BIO_READ) {
 987                         if (beio->beio_cont == NULL &&
 988                             cbe_lun->serseq == CTL_LUN_SERSEQ_SOFT)
 989                                 ctl_serseq_done(io);
 990                         error = csw->d_read(dev, &xuio, flags);
 991                 } else
 992                         error = csw->d_write(dev, &xuio, flags);
 993                 dev_relthread(dev, ref);
 994         } else
 995                 error = ENXIO;
 996
 997         if (beio->bio_cmd == BIO_READ)
 998                 SDT_PROBE0(cbb, , read, file_done);
 999         else
1000                 SDT_PROBE0(cbb, , write, file_done);
1001
1002         mtx_lock(&be_lun->io_lock);
1003         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
1004             beio->ds_tag_type, beio->ds_trans_type,
1005             /*now*/ NULL, /*then*/&beio->ds_t0);
1006         mtx_unlock(&be_lun->io_lock);
1007
1008         /*
1009          * If we got an error, set the sense data to "MEDIUM ERROR" and
1010          * return the I/O to the user.
1011          */
1012         if (error != 0) {
1013                 if (error == ENOSPC || error == EDQUOT) {
1014                         ctl_set_space_alloc_fail(&io->scsiio);
1015                 } else if (error == EROFS || error == EACCES) {
1016                         ctl_set_hw_write_protected(&io->scsiio);
1017                 } else {
1018                         ctl_set_medium_error(&io->scsiio,
1019                             beio->bio_cmd == BIO_READ);
1020                 }
1021                 ctl_complete_beio(beio);
1022                 return;
1023         }
1024
1025         /*
1026          * If this is a write or a verify, we're all done.
1027          * If this is a read, we can now send the data to the user.
1028          */
1029         if ((beio->bio_cmd == BIO_WRITE) ||
1030             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
1031                 ctl_set_success(&io->scsiio);
1032                 ctl_complete_beio(beio);
1033         } else {
1034                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
1035                     beio->beio_cont == NULL) {
1036                         ctl_set_success(&io->scsiio);
1037                         if (cbe_lun->serseq > CTL_LUN_SERSEQ_SOFT)
1038                                 ctl_serseq_done(io);
1039                 }
1040                 ctl_datamove(io);
1041         }
1042 }
1043
1044 static void
1045 ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun,
1046                         struct ctl_be_block_io *beio)
1047 {
1048         union ctl_io *io = beio->io;
1049         struct cdevsw *csw;
1050         struct cdev *dev;
1051         struct ctl_lba_len_flags *lbalen = ARGS(io);
1052         struct scsi_get_lba_status_data *data;
1053         off_t roff, off;
1054         int error, ref, status;
1055
1056         DPRINTF("entered\n");
1057
1058         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1059         if (csw == NULL) {
1060                 status = 0;     /* unknown up to the end */
1061                 off = be_lun->size_bytes;
1062                 goto done;
1063         }
1064         off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
1065         error = csw->d_ioctl(dev, FIOSEEKHOLE, (caddr_t)&off, FREAD,
1066             curthread);
1067         if (error == 0 && off > roff)
1068                 status = 0;     /* mapped up to off */
1069         else {
1070                 error = csw->d_ioctl(dev, FIOSEEKDATA, (caddr_t)&off, FREAD,
1071                     curthread);
1072                 if (error == 0 && off > roff)
1073                         status = 1;     /* deallocated up to off */
1074                 else {
1075                         status = 0;     /* unknown up to the end */
1076                         off = be_lun->size_bytes;
1077                 }
1078         }
1079         dev_relthread(dev, ref);
1080
1081 done:
1082         data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
1083         scsi_u64to8b(lbalen->lba, data->descr[0].addr);
1084         scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
1085             lbalen->lba), data->descr[0].length);
1086         data->descr[0].status = status;
1087
1088         ctl_complete_beio(beio);
1089 }
1090
1091 static void
1092 ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
1093                        struct ctl_be_block_io *beio)
1094 {
1095         struct bio *bio;
1096         struct cdevsw *csw;
1097         struct cdev *dev;
1098         int ref;
1099
1100         DPRINTF("entered\n");
1101
1102         /* This can't fail, it's a blocking allocation. */
1103         bio = g_alloc_bio();
1104
1105         bio->bio_cmd        = BIO_FLUSH;
1106         bio->bio_offset     = 0;
1107         bio->bio_data       = 0;
1108         bio->bio_done       = ctl_be_block_biodone;
1109         bio->bio_caller1    = beio;
1110         bio->bio_pblkno     = 0;
1111
1112         /*
1113          * We don't need to acquire the LUN lock here, because we are only
1114          * sending one bio, and so there is no other context to synchronize
1115          * with.
1116          */
1117         beio->num_bios_sent = 1;
1118         beio->send_complete = 1;
1119
1120         binuptime(&beio->ds_t0);
1121         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1122
1123         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1124         if (csw) {
1125                 bio->bio_dev = dev;
1126                 csw->d_strategy(bio);
1127                 dev_relthread(dev, ref);
1128         } else {
1129                 bio->bio_error = ENXIO;
1130                 ctl_be_block_biodone(bio);
1131         }
1132 }
1133
1134 static void
1135 ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun,
1136                        struct ctl_be_block_io *beio,
1137                        uint64_t off, uint64_t len, int last)
1138 {
1139         struct bio *bio;
1140         uint64_t maxlen;
1141         struct cdevsw *csw;
1142         struct cdev *dev;
1143         int ref;
1144
1145         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1146         maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize);
1147         while (len > 0) {
1148                 bio = g_alloc_bio();
1149                 bio->bio_cmd        = BIO_DELETE;
1150                 bio->bio_dev        = dev;
1151                 bio->bio_offset     = off;
1152                 bio->bio_length     = MIN(len, maxlen);
1153                 bio->bio_data       = 0;
1154                 bio->bio_done       = ctl_be_block_biodone;
1155                 bio->bio_caller1    = beio;
1156                 bio->bio_pblkno     = off / be_lun->cbe_lun.blocksize;
1157
1158                 off += bio->bio_length;
1159                 len -= bio->bio_length;
1160
1161                 mtx_lock(&be_lun->io_lock);
1162                 beio->num_bios_sent++;
1163                 if (last && len == 0)
1164                         beio->send_complete = 1;
1165                 mtx_unlock(&be_lun->io_lock);
1166
1167                 if (csw) {
1168                         csw->d_strategy(bio);
1169                 } else {
1170                         bio->bio_error = ENXIO;
1171                         ctl_be_block_biodone(bio);
1172                 }
1173         }
1174         if (csw)
1175                 dev_relthread(dev, ref);
1176 }
1177
1178 static void
1179 ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
1180                        struct ctl_be_block_io *beio)
1181 {
1182         union ctl_io *io;
1183         struct ctl_ptr_len_flags *ptrlen;
1184         struct scsi_unmap_desc *buf, *end;
1185         uint64_t len;
1186
1187         io = beio->io;
1188
1189         DPRINTF("entered\n");
1190
1191         binuptime(&beio->ds_t0);
1192         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1193
1194         if (beio->io_offset == -1) {
1195                 beio->io_len = 0;
1196                 ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1197                 buf = (struct scsi_unmap_desc *)ptrlen->ptr;
1198                 end = buf + ptrlen->len / sizeof(*buf);
1199                 for (; buf < end; buf++) {
1200                         len = (uint64_t)scsi_4btoul(buf->length) *
1201                             be_lun->cbe_lun.blocksize;
1202                         beio->io_len += len;
1203                         ctl_be_block_unmap_dev_range(be_lun, beio,
1204                             scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize,
1205                             len, (end - buf < 2) ? TRUE : FALSE);
1206                 }
1207         } else
1208                 ctl_be_block_unmap_dev_range(be_lun, beio,
1209                     beio->io_offset, beio->io_len, TRUE);
1210 }
1211
1212 static void
1213 ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
1214                           struct ctl_be_block_io *beio)
1215 {
1216         TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue);
1217         struct bio *bio;
1218         struct cdevsw *csw;
1219         struct cdev *dev;
1220         off_t cur_offset;
1221         int i, max_iosize, ref;
1222
1223         DPRINTF("entered\n");
1224         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1225
1226         /*
1227          * We have to limit our I/O size to the maximum supported by the
1228          * backend device.
1229          */
1230         if (csw) {
1231                 max_iosize = dev->si_iosize_max;
1232                 if (max_iosize <= 0)
1233                         max_iosize = DFLTPHYS;
1234         } else
1235                 max_iosize = maxphys;
1236
1237         cur_offset = beio->io_offset;
1238         for (i = 0; i < beio->num_segs; i++) {
1239                 size_t cur_size;
1240                 uint8_t *cur_ptr;
1241
1242                 cur_size = beio->sg_segs[i].len;
1243                 cur_ptr = beio->sg_segs[i].addr;
1244
1245                 while (cur_size > 0) {
1246                         /* This can't fail, it's a blocking allocation. */
1247                         bio = g_alloc_bio();
1248
1249                         KASSERT(bio != NULL, ("g_alloc_bio() failed!\n"));
1250
1251                         bio->bio_cmd = beio->bio_cmd;
1252                         bio->bio_dev = dev;
1253                         bio->bio_caller1 = beio;
1254                         bio->bio_length = min(cur_size, max_iosize);
1255                         bio->bio_offset = cur_offset;
1256                         bio->bio_data = cur_ptr;
1257                         bio->bio_done = ctl_be_block_biodone;
1258                         bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize;
1259
1260                         cur_offset += bio->bio_length;
1261                         cur_ptr += bio->bio_length;
1262                         cur_size -= bio->bio_length;
1263
1264                         TAILQ_INSERT_TAIL(&queue, bio, bio_queue);
1265                         beio->num_bios_sent++;
1266                 }
1267         }
1268         beio->send_complete = 1;
1269         binuptime(&beio->ds_t0);
1270         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1271
1272         /*
1273          * Fire off all allocated requests!
1274          */
1275         while ((bio = TAILQ_FIRST(&queue)) != NULL) {
1276                 TAILQ_REMOVE(&queue, bio, bio_queue);
1277                 if (csw)
1278                         csw->d_strategy(bio);
1279                 else {
1280                         bio->bio_error = ENXIO;
1281                         ctl_be_block_biodone(bio);
1282                 }
1283         }
1284         if (csw)
1285                 dev_relthread(dev, ref);
1286 }
1287
1288 static uint64_t
1289 ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname)
1290 {
1291         struct diocgattr_arg    arg;
1292         struct cdevsw *csw;
1293         struct cdev *dev;
1294         int error, ref;
1295
1296         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1297         if (csw == NULL)
1298                 return (UINT64_MAX);
1299         strlcpy(arg.name, attrname, sizeof(arg.name));
1300         arg.len = sizeof(arg.value.off);
1301         if (csw->d_ioctl) {
1302                 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
1303                     curthread);
1304         } else
1305                 error = ENODEV;
1306         dev_relthread(dev, ref);
1307         if (error != 0)
1308                 return (UINT64_MAX);
1309         return (arg.value.off);
1310 }
1311
1312 static void
1313 ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun,
1314                             union ctl_io *io)
1315 {
1316         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1317         struct ctl_be_block_io *beio;
1318         struct ctl_lba_len_flags *lbalen;
1319
1320         DPRINTF("entered\n");
1321         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1322         lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1323
1324         beio->io_len = lbalen->len * cbe_lun->blocksize;
1325         beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1326         beio->io_arg = (lbalen->flags & SSC_IMMED) != 0;
1327         beio->bio_cmd = BIO_FLUSH;
1328         beio->ds_trans_type = DEVSTAT_NO_DATA;
1329         DPRINTF("SYNC\n");
1330         be_lun->lun_flush(be_lun, beio);
1331 }
1332
1333 static void
1334 ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio)
1335 {
1336         union ctl_io *io;
1337
1338         io = beio->io;
1339         ctl_free_beio(beio);
1340         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1341             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1342              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1343                 ctl_config_write_done(io);
1344                 return;
1345         }
1346
1347         ctl_be_block_config_write(io);
1348 }
1349
1350 static void
1351 ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,
1352                             union ctl_io *io)
1353 {
1354         struct ctl_be_block_softc *softc = be_lun->softc;
1355         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1356         struct ctl_be_block_io *beio;
1357         struct ctl_lba_len_flags *lbalen;
1358         uint64_t len_left, lba;
1359         uint32_t pb, pbo, adj;
1360         int i, seglen;
1361         uint8_t *buf, *end;
1362
1363         DPRINTF("entered\n");
1364
1365         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1366         lbalen = ARGS(io);
1367
1368         if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) ||
1369             (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) {
1370                 ctl_free_beio(beio);
1371                 ctl_set_invalid_field(&io->scsiio,
1372                                       /*sks_valid*/ 1,
1373                                       /*command*/ 1,
1374                                       /*field*/ 1,
1375                                       /*bit_valid*/ 0,
1376                                       /*bit*/ 0);
1377                 ctl_config_write_done(io);
1378                 return;
1379         }
1380
1381         if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) {
1382                 beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1383                 beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize;
1384                 beio->bio_cmd = BIO_DELETE;
1385                 beio->ds_trans_type = DEVSTAT_FREE;
1386
1387                 be_lun->unmap(be_lun, beio);
1388                 return;
1389         }
1390
1391         beio->bio_cmd = BIO_WRITE;
1392         beio->ds_trans_type = DEVSTAT_WRITE;
1393
1394         DPRINTF("WRITE SAME at LBA %jx len %u\n",
1395                (uintmax_t)lbalen->lba, lbalen->len);
1396
1397         pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp;
1398         if (be_lun->cbe_lun.pblockoff > 0)
1399                 pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff;
1400         else
1401                 pbo = 0;
1402         len_left = (uint64_t)lbalen->len * cbe_lun->blocksize;
1403         for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) {
1404                 /*
1405                  * Setup the S/G entry for this chunk.
1406                  */
1407                 seglen = MIN(CTLBLK_MAX_SEG, len_left);
1408                 if (pb > cbe_lun->blocksize) {
1409                         adj = ((lbalen->lba + lba) * cbe_lun->blocksize +
1410                             seglen - pbo) % pb;
1411                         if (seglen > adj)
1412                                 seglen -= adj;
1413                         else
1414                                 seglen -= seglen % cbe_lun->blocksize;
1415                 } else
1416                         seglen -= seglen % cbe_lun->blocksize;
1417                 ctl_alloc_seg(softc, &beio->sg_segs[i], seglen);
1418
1419                 DPRINTF("segment %d addr %p len %zd\n", i,
1420                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1421
1422                 beio->num_segs++;
1423                 len_left -= seglen;
1424
1425                 buf = beio->sg_segs[i].addr;
1426                 end = buf + seglen;
1427                 for (; buf < end; buf += cbe_lun->blocksize) {
1428                         if (lbalen->flags & SWS_NDOB) {
1429                                 memset(buf, 0, cbe_lun->blocksize);
1430                         } else {
1431                                 memcpy(buf, io->scsiio.kern_data_ptr,
1432                                     cbe_lun->blocksize);
1433                         }
1434                         if (lbalen->flags & SWS_LBDATA)
1435                                 scsi_ulto4b(lbalen->lba + lba, buf);
1436                         lba++;
1437                 }
1438         }
1439
1440         beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1441         beio->io_len = lba * cbe_lun->blocksize;
1442
1443         /* We can not do all in one run. Correct and schedule rerun. */
1444         if (len_left > 0) {
1445                 lbalen->lba += lba;
1446                 lbalen->len -= lba;
1447                 beio->beio_cont = ctl_be_block_cw_done_ws;
1448         }
1449
1450         be_lun->dispatch(be_lun, beio);
1451 }
1452
1453 static void
1454 ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun,
1455                             union ctl_io *io)
1456 {
1457         struct ctl_be_block_io *beio;
1458         struct ctl_ptr_len_flags *ptrlen;
1459
1460         DPRINTF("entered\n");
1461
1462         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1463         ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1464
1465         if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) {
1466                 ctl_free_beio(beio);
1467                 ctl_set_invalid_field(&io->scsiio,
1468                                       /*sks_valid*/ 0,
1469                                       /*command*/ 1,
1470                                       /*field*/ 0,
1471                                       /*bit_valid*/ 0,
1472                                       /*bit*/ 0);
1473                 ctl_config_write_done(io);
1474                 return;
1475         }
1476
1477         beio->io_len = 0;
1478         beio->io_offset = -1;
1479         beio->bio_cmd = BIO_DELETE;
1480         beio->ds_trans_type = DEVSTAT_FREE;
1481         DPRINTF("UNMAP\n");
1482         be_lun->unmap(be_lun, beio);
1483 }
1484
1485 static void
1486 ctl_be_block_cr_done(struct ctl_be_block_io *beio)
1487 {
1488         union ctl_io *io;
1489
1490         io = beio->io;
1491         ctl_free_beio(beio);
1492         ctl_config_read_done(io);
1493 }
1494
1495 static void
1496 ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
1497                          union ctl_io *io)
1498 {
1499         struct ctl_be_block_io *beio;
1500         struct ctl_be_block_softc *softc;
1501
1502         DPRINTF("entered\n");
1503
1504         softc = be_lun->softc;
1505         beio = ctl_alloc_beio(softc);
1506         beio->io = io;
1507         beio->lun = be_lun;
1508         beio->beio_cont = ctl_be_block_cr_done;
1509         PRIV(io)->ptr = (void *)beio;
1510
1511         switch (io->scsiio.cdb[0]) {
1512         case SERVICE_ACTION_IN:         /* GET LBA STATUS */
1513                 beio->bio_cmd = -1;
1514                 beio->ds_trans_type = DEVSTAT_NO_DATA;
1515                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1516                 beio->io_len = 0;
1517                 if (be_lun->get_lba_status)
1518                         be_lun->get_lba_status(be_lun, beio);
1519                 else
1520                         ctl_be_block_cr_done(beio);
1521                 break;
1522         default:
1523                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1524                 break;
1525         }
1526 }
1527
1528 static void
1529 ctl_be_block_cw_done(struct ctl_be_block_io *beio)
1530 {
1531         union ctl_io *io;
1532
1533         io = beio->io;
1534         ctl_free_beio(beio);
1535         ctl_config_write_done(io);
1536 }
1537
1538 static void
1539 ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
1540                          union ctl_io *io)
1541 {
1542         struct ctl_be_block_io *beio;
1543         struct ctl_be_block_softc *softc;
1544
1545         DPRINTF("entered\n");
1546
1547         softc = be_lun->softc;
1548         beio = ctl_alloc_beio(softc);
1549         beio->io = io;
1550         beio->lun = be_lun;
1551         beio->beio_cont = ctl_be_block_cw_done;
1552         switch (io->scsiio.tag_type) {
1553         case CTL_TAG_ORDERED:
1554                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1555                 break;
1556         case CTL_TAG_HEAD_OF_QUEUE:
1557                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1558                 break;
1559         case CTL_TAG_UNTAGGED:
1560         case CTL_TAG_SIMPLE:
1561         case CTL_TAG_ACA:
1562         default:
1563                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1564                 break;
1565         }
1566         PRIV(io)->ptr = (void *)beio;
1567
1568         switch (io->scsiio.cdb[0]) {
1569         case SYNCHRONIZE_CACHE:
1570         case SYNCHRONIZE_CACHE_16:
1571                 ctl_be_block_cw_dispatch_sync(be_lun, io);
1572                 break;
1573         case WRITE_SAME_10:
1574         case WRITE_SAME_16:
1575                 ctl_be_block_cw_dispatch_ws(be_lun, io);
1576                 break;
1577         case UNMAP:
1578                 ctl_be_block_cw_dispatch_unmap(be_lun, io);
1579                 break;
1580         default:
1581                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1582                 break;
1583         }
1584 }
1585
1586 SDT_PROBE_DEFINE1(cbb, , read, start, "uint64_t");
1587 SDT_PROBE_DEFINE1(cbb, , write, start, "uint64_t");
1588 SDT_PROBE_DEFINE1(cbb, , read, alloc_done, "uint64_t");
1589 SDT_PROBE_DEFINE1(cbb, , write, alloc_done, "uint64_t");
1590
1591 static void
1592 ctl_be_block_next(struct ctl_be_block_io *beio)
1593 {
1594         struct ctl_be_block_lun *be_lun;
1595         union ctl_io *io;
1596
1597         io = beio->io;
1598         be_lun = beio->lun;
1599         ctl_free_beio(beio);
1600         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1601             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1602              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1603                 ctl_data_submit_done(io);
1604                 return;
1605         }
1606
1607         io->io_hdr.status &= ~CTL_STATUS_MASK;
1608         io->io_hdr.status |= CTL_STATUS_NONE;
1609
1610         mtx_lock(&be_lun->queue_lock);
1611         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1612         mtx_unlock(&be_lun->queue_lock);
1613         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1614 }
1615
1616 static void
1617 ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
1618                            union ctl_io *io)
1619 {
1620         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1621         struct ctl_be_block_io *beio;
1622         struct ctl_be_block_softc *softc;
1623         struct ctl_lba_len_flags *lbalen;
1624         struct ctl_ptr_len_flags *bptrlen;
1625         uint64_t len_left, lbas;
1626         int i;
1627
1628         softc = be_lun->softc;
1629
1630         DPRINTF("entered\n");
1631
1632         lbalen = ARGS(io);
1633         if (lbalen->flags & CTL_LLF_WRITE) {
1634                 SDT_PROBE0(cbb, , write, start);
1635         } else {
1636                 SDT_PROBE0(cbb, , read, start);
1637         }
1638
1639         beio = ctl_alloc_beio(softc);
1640         beio->io = io;
1641         beio->lun = be_lun;
1642         bptrlen = PRIV(io);
1643         bptrlen->ptr = (void *)beio;
1644
1645         switch (io->scsiio.tag_type) {
1646         case CTL_TAG_ORDERED:
1647                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1648                 break;
1649         case CTL_TAG_HEAD_OF_QUEUE:
1650                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1651                 break;
1652         case CTL_TAG_UNTAGGED:
1653         case CTL_TAG_SIMPLE:
1654         case CTL_TAG_ACA:
1655         default:
1656                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1657                 break;
1658         }
1659
1660         if (lbalen->flags & CTL_LLF_WRITE) {
1661                 beio->bio_cmd = BIO_WRITE;
1662                 beio->ds_trans_type = DEVSTAT_WRITE;
1663         } else {
1664                 beio->bio_cmd = BIO_READ;
1665                 beio->ds_trans_type = DEVSTAT_READ;
1666         }
1667
1668         DPRINTF("%s at LBA %jx len %u @%ju\n",
1669                (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
1670                (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len);
1671         lbas = CTLBLK_MAX_IO_SIZE;
1672         if (lbalen->flags & CTL_LLF_COMPARE) {
1673                 beio->two_sglists = 1;
1674                 lbas /= 2;
1675         }
1676         lbas = MIN(lbalen->len - bptrlen->len, lbas / cbe_lun->blocksize);
1677         beio->io_offset = (lbalen->lba + bptrlen->len) * cbe_lun->blocksize;
1678         beio->io_len = lbas * cbe_lun->blocksize;
1679         bptrlen->len += lbas;
1680
1681         for (i = 0, len_left = beio->io_len; len_left > 0; i++) {
1682                 KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)",
1683                     i, CTLBLK_MAX_SEGS));
1684
1685                 /*
1686                  * Setup the S/G entry for this chunk.
1687                  */
1688                 ctl_alloc_seg(softc, &beio->sg_segs[i],
1689                     MIN(CTLBLK_MAX_SEG, len_left));
1690
1691                 DPRINTF("segment %d addr %p len %zd\n", i,
1692                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1693
1694                 /* Set up second segment for compare operation. */
1695                 if (beio->two_sglists) {
1696                         ctl_alloc_seg(softc,
1697                             &beio->sg_segs[i + CTLBLK_HALF_SEGS],
1698                             beio->sg_segs[i].len);
1699                 }
1700
1701                 beio->num_segs++;
1702                 len_left -= beio->sg_segs[i].len;
1703         }
1704         if (bptrlen->len < lbalen->len)
1705                 beio->beio_cont = ctl_be_block_next;
1706         io->scsiio.be_move_done = ctl_be_block_move_done;
1707         /* For compare we have separate S/G lists for read and datamove. */
1708         if (beio->two_sglists)
1709                 io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS];
1710         else
1711                 io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
1712         io->scsiio.kern_data_len = beio->io_len;
1713         io->scsiio.kern_sg_entries = beio->num_segs;
1714         io->scsiio.kern_data_ref = ctl_refcnt_beio;
1715         io->scsiio.kern_data_arg = beio;
1716         io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
1717
1718         /*
1719          * For the read case, we need to read the data into our buffers and
1720          * then we can send it back to the user.  For the write case, we
1721          * need to get the data from the user first.
1722          */
1723         if (beio->bio_cmd == BIO_READ) {
1724                 SDT_PROBE0(cbb, , read, alloc_done);
1725                 be_lun->dispatch(be_lun, beio);
1726         } else {
1727                 SDT_PROBE0(cbb, , write, alloc_done);
1728                 ctl_datamove(io);
1729         }
1730 }
1731
1732 static void
1733 ctl_be_block_worker(void *context, int pending)
1734 {
1735         struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context;
1736         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1737         union ctl_io *io;
1738         struct ctl_be_block_io *beio;
1739
1740         DPRINTF("entered\n");
1741         /*
1742          * Fetch and process I/Os from all queues.  If we detect LUN
1743          * CTL_LUN_FLAG_NO_MEDIA status here -- it is result of a race,
1744          * so make response maximally opaque to not confuse initiator.
1745          */
1746         for (;;) {
1747                 mtx_lock(&be_lun->queue_lock);
1748                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);
1749                 if (io != NULL) {
1750                         DPRINTF("datamove queue\n");
1751                         STAILQ_REMOVE_HEAD(&be_lun->datamove_queue, links);
1752                         mtx_unlock(&be_lun->queue_lock);
1753                         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1754                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1755                                 ctl_set_busy(&io->scsiio);
1756                                 ctl_complete_beio(beio);
1757                                 continue;
1758                         }
1759                         be_lun->dispatch(be_lun, beio);
1760                         continue;
1761                 }
1762                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);
1763                 if (io != NULL) {
1764                         DPRINTF("config write queue\n");
1765                         STAILQ_REMOVE_HEAD(&be_lun->config_write_queue, links);
1766                         mtx_unlock(&be_lun->queue_lock);
1767                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1768                                 ctl_set_busy(&io->scsiio);
1769                                 ctl_config_write_done(io);
1770                                 continue;
1771                         }
1772                         ctl_be_block_cw_dispatch(be_lun, io);
1773                         continue;
1774                 }
1775                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue);
1776                 if (io != NULL) {
1777                         DPRINTF("config read queue\n");
1778                         STAILQ_REMOVE_HEAD(&be_lun->config_read_queue, links);
1779                         mtx_unlock(&be_lun->queue_lock);
1780                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1781                                 ctl_set_busy(&io->scsiio);
1782                                 ctl_config_read_done(io);
1783                                 continue;
1784                         }
1785                         ctl_be_block_cr_dispatch(be_lun, io);
1786                         continue;
1787                 }
1788                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);
1789                 if (io != NULL) {
1790                         DPRINTF("input queue\n");
1791                         STAILQ_REMOVE_HEAD(&be_lun->input_queue, links);
1792                         mtx_unlock(&be_lun->queue_lock);
1793                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1794                                 ctl_set_busy(&io->scsiio);
1795                                 ctl_data_submit_done(io);
1796                                 continue;
1797                         }
1798                         ctl_be_block_dispatch(be_lun, io);
1799                         continue;
1800                 }
1801
1802                 /*
1803                  * If we get here, there is no work left in the queues, so
1804                  * just break out and let the task queue go to sleep.
1805                  */
1806                 mtx_unlock(&be_lun->queue_lock);
1807                 break;
1808         }
1809 }
1810
1811 /*
1812  * Entry point from CTL to the backend for I/O.  We queue everything to a
1813  * work thread, so this just puts the I/O on a queue and wakes up the
1814  * thread.
1815  */
1816 static int
1817 ctl_be_block_submit(union ctl_io *io)
1818 {
1819         struct ctl_be_block_lun *be_lun;
1820
1821         DPRINTF("entered\n");
1822
1823         be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);
1824
1825         KASSERT(io->io_hdr.io_type == CTL_IO_SCSI,
1826             ("%s: unexpected I/O type %x", __func__, io->io_hdr.io_type));
1827
1828         PRIV(io)->len = 0;
1829
1830         mtx_lock(&be_lun->queue_lock);
1831         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1832         mtx_unlock(&be_lun->queue_lock);
1833         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1834
1835         return (CTL_RETVAL_COMPLETE);
1836 }
1837
1838 static int
1839 ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
1840                         int flag, struct thread *td)
1841 {
1842         struct ctl_be_block_softc *softc = &backend_block_softc;
1843         int error;
1844
1845         error = 0;
1846         switch (cmd) {
1847         case CTL_LUN_REQ: {
1848                 struct ctl_lun_req *lun_req;
1849
1850                 lun_req = (struct ctl_lun_req *)addr;
1851
1852                 switch (lun_req->reqtype) {
1853                 case CTL_LUNREQ_CREATE:
1854                         error = ctl_be_block_create(softc, lun_req);
1855                         break;
1856                 case CTL_LUNREQ_RM:
1857                         error = ctl_be_block_rm(softc, lun_req);
1858                         break;
1859                 case CTL_LUNREQ_MODIFY:
1860                         error = ctl_be_block_modify(softc, lun_req);
1861                         break;
1862                 default:
1863                         lun_req->status = CTL_LUN_ERROR;
1864                         snprintf(lun_req->error_str, sizeof(lun_req->error_str),
1865                                  "invalid LUN request type %d",
1866                                  lun_req->reqtype);
1867                         break;
1868                 }
1869                 break;
1870         }
1871         default:
1872                 error = ENOTTY;
1873                 break;
1874         }
1875
1876         return (error);
1877 }
1878
1879 static int
1880 ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1881 {
1882         struct ctl_be_lun *cbe_lun;
1883         struct ctl_be_block_filedata *file_data;
1884         struct ctl_lun_create_params *params;
1885         const char                   *value;
1886         struct vattr                  vattr;
1887         off_t                         ps, pss, po, pos, us, uss, uo, uos;
1888         int                           error;
1889         long                          pconf;
1890
1891         cbe_lun = &be_lun->cbe_lun;
1892         file_data = &be_lun->backend.file;
1893         params = &be_lun->params;
1894
1895         be_lun->dev_type = CTL_BE_BLOCK_FILE;
1896         be_lun->dispatch = ctl_be_block_dispatch_file;
1897         be_lun->lun_flush = ctl_be_block_flush_file;
1898         be_lun->get_lba_status = ctl_be_block_gls_file;
1899         be_lun->getattr = ctl_be_block_getattr_file;
1900         be_lun->unmap = ctl_be_block_unmap_file;
1901         cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
1902
1903         error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
1904         if (error != 0) {
1905                 snprintf(req->error_str, sizeof(req->error_str),
1906                          "error calling VOP_GETATTR() for file %s",
1907                          be_lun->dev_path);
1908                 return (error);
1909         }
1910
1911         error = VOP_PATHCONF(be_lun->vn, _PC_DEALLOC_PRESENT, &pconf);
1912         if (error != 0) {
1913                 snprintf(req->error_str, sizeof(req->error_str),
1914                     "error calling VOP_PATHCONF() for file %s",
1915                     be_lun->dev_path);
1916                 return (error);
1917         }
1918         if (pconf == 1)
1919                 cbe_lun->flags |= CTL_LUN_FLAG_UNMAP;
1920
1921         file_data->cred = crhold(curthread->td_ucred);
1922         if (params->lun_size_bytes != 0)
1923                 be_lun->size_bytes = params->lun_size_bytes;
1924         else
1925                 be_lun->size_bytes = vattr.va_size;
1926
1927         /*
1928          * For files we can use any logical block size.  Prefer 512 bytes
1929          * for compatibility reasons.  If file's vattr.va_blocksize
1930          * (preferred I/O block size) is bigger and multiple to chosen
1931          * logical block size -- report it as physical block size.
1932          */
1933         if (params->blocksize_bytes != 0)
1934                 cbe_lun->blocksize = params->blocksize_bytes;
1935         else if (cbe_lun->lun_type == T_CDROM)
1936                 cbe_lun->blocksize = 2048;
1937         else
1938                 cbe_lun->blocksize = 512;
1939         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
1940         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
1941             0 : (be_lun->size_blocks - 1);
1942
1943         us = ps = vattr.va_blocksize;
1944         uo = po = 0;
1945
1946         value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL);
1947         if (value != NULL)
1948                 ctl_expand_number(value, &ps);
1949         value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL);
1950         if (value != NULL)
1951                 ctl_expand_number(value, &po);
1952         pss = ps / cbe_lun->blocksize;
1953         pos = po / cbe_lun->blocksize;
1954         if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
1955             ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
1956                 cbe_lun->pblockexp = fls(pss) - 1;
1957                 cbe_lun->pblockoff = (pss - pos) % pss;
1958         }
1959
1960         value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL);
1961         if (value != NULL)
1962                 ctl_expand_number(value, &us);
1963         value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL);
1964         if (value != NULL)
1965                 ctl_expand_number(value, &uo);
1966         uss = us / cbe_lun->blocksize;
1967         uos = uo / cbe_lun->blocksize;
1968         if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
1969             ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
1970                 cbe_lun->ublockexp = fls(uss) - 1;
1971                 cbe_lun->ublockoff = (uss - uos) % uss;
1972         }
1973
1974         /*
1975          * Sanity check.  The media size has to be at least one
1976          * sector long.
1977          */
1978         if (be_lun->size_bytes < cbe_lun->blocksize) {
1979                 error = EINVAL;
1980                 snprintf(req->error_str, sizeof(req->error_str),
1981                          "file %s size %ju < block size %u", be_lun->dev_path,
1982                          (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize);
1983         }
1984
1985         cbe_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / cbe_lun->blocksize;
1986         return (error);
1987 }
1988
1989 static int
1990 ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1991 {
1992         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1993         struct ctl_lun_create_params *params;
1994         struct cdevsw                *csw;
1995         struct cdev                  *dev;
1996         const char                   *value;
1997         int                           error, atomic, maxio, ref, unmap, tmp;
1998         off_t                         ps, pss, po, pos, us, uss, uo, uos, otmp;
1999
2000         params = &be_lun->params;
2001
2002         be_lun->dev_type = CTL_BE_BLOCK_DEV;
2003         csw = devvn_refthread(be_lun->vn, &dev, &ref);
2004         if (csw == NULL)
2005                 return (ENXIO);
2006         if (strcmp(csw->d_name, "zvol") == 0) {
2007                 be_lun->dispatch = ctl_be_block_dispatch_zvol;
2008                 be_lun->get_lba_status = ctl_be_block_gls_zvol;
2009                 atomic = maxio = CTLBLK_MAX_IO_SIZE;
2010         } else {
2011                 be_lun->dispatch = ctl_be_block_dispatch_dev;
2012                 be_lun->get_lba_status = NULL;
2013                 atomic = 0;
2014                 maxio = dev->si_iosize_max;
2015                 if (maxio <= 0)
2016                         maxio = DFLTPHYS;
2017                 if (maxio > CTLBLK_MAX_SEG)
2018                         maxio = CTLBLK_MAX_SEG;
2019         }
2020         be_lun->lun_flush = ctl_be_block_flush_dev;
2021         be_lun->getattr = ctl_be_block_getattr_dev;
2022         be_lun->unmap = ctl_be_block_unmap_dev;
2023
2024         if (!csw->d_ioctl) {
2025                 dev_relthread(dev, ref);
2026                 snprintf(req->error_str, sizeof(req->error_str),
2027                          "no d_ioctl for device %s!", be_lun->dev_path);
2028                 return (ENODEV);
2029         }
2030
2031         error = csw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD,
2032                                curthread);
2033         if (error) {
2034                 dev_relthread(dev, ref);
2035                 snprintf(req->error_str, sizeof(req->error_str),
2036                          "error %d returned for DIOCGSECTORSIZE ioctl "
2037                          "on %s!", error, be_lun->dev_path);
2038                 return (error);
2039         }
2040
2041         /*
2042          * If the user has asked for a blocksize that is greater than the
2043          * backing device's blocksize, we can do it only if the blocksize
2044          * the user is asking for is an even multiple of the underlying
2045          * device's blocksize.
2046          */
2047         if ((params->blocksize_bytes != 0) &&
2048             (params->blocksize_bytes >= tmp)) {
2049                 if (params->blocksize_bytes % tmp == 0) {
2050                         cbe_lun->blocksize = params->blocksize_bytes;
2051                 } else {
2052                         dev_relthread(dev, ref);
2053                         snprintf(req->error_str, sizeof(req->error_str),
2054                                  "requested blocksize %u is not an even "
2055                                  "multiple of backing device blocksize %u",
2056                                  params->blocksize_bytes, tmp);
2057                         return (EINVAL);
2058                 }
2059         } else if (params->blocksize_bytes != 0) {
2060                 dev_relthread(dev, ref);
2061                 snprintf(req->error_str, sizeof(req->error_str),
2062                          "requested blocksize %u < backing device "
2063                          "blocksize %u", params->blocksize_bytes, tmp);
2064                 return (EINVAL);
2065         } else if (cbe_lun->lun_type == T_CDROM)
2066                 cbe_lun->blocksize = MAX(tmp, 2048);
2067         else
2068                 cbe_lun->blocksize = tmp;
2069
2070         error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD,
2071                              curthread);
2072         if (error) {
2073                 dev_relthread(dev, ref);
2074                 snprintf(req->error_str, sizeof(req->error_str),
2075                          "error %d returned for DIOCGMEDIASIZE "
2076                          " ioctl on %s!", error,
2077                          be_lun->dev_path);
2078                 return (error);
2079         }
2080
2081         if (params->lun_size_bytes != 0) {
2082                 if (params->lun_size_bytes > otmp) {
2083                         dev_relthread(dev, ref);
2084                         snprintf(req->error_str, sizeof(req->error_str),
2085                                  "requested LUN size %ju > backing device "
2086                                  "size %ju",
2087                                  (uintmax_t)params->lun_size_bytes,
2088                                  (uintmax_t)otmp);
2089                         return (EINVAL);
2090                 }
2091
2092                 be_lun->size_bytes = params->lun_size_bytes;
2093         } else
2094                 be_lun->size_bytes = otmp;
2095         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
2096         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
2097             0 : (be_lun->size_blocks - 1);
2098
2099         error = csw->d_ioctl(dev, DIOCGSTRIPESIZE, (caddr_t)&ps, FREAD,
2100             curthread);
2101         if (error)
2102                 ps = po = 0;
2103         else {
2104                 error = csw->d_ioctl(dev, DIOCGSTRIPEOFFSET, (caddr_t)&po,
2105                     FREAD, curthread);
2106                 if (error)
2107                         po = 0;
2108         }
2109         us = ps;
2110         uo = po;
2111
2112         value = dnvlist_get_string(cbe_lun->options, "pblocksize", NULL);
2113         if (value != NULL)
2114                 ctl_expand_number(value, &ps);
2115         value = dnvlist_get_string(cbe_lun->options, "pblockoffset", NULL);
2116         if (value != NULL)
2117                 ctl_expand_number(value, &po);
2118         pss = ps / cbe_lun->blocksize;
2119         pos = po / cbe_lun->blocksize;
2120         if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
2121             ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
2122                 cbe_lun->pblockexp = fls(pss) - 1;
2123                 cbe_lun->pblockoff = (pss - pos) % pss;
2124         }
2125
2126         value = dnvlist_get_string(cbe_lun->options, "ublocksize", NULL);
2127         if (value != NULL)
2128                 ctl_expand_number(value, &us);
2129         value = dnvlist_get_string(cbe_lun->options, "ublockoffset", NULL);
2130         if (value != NULL)
2131                 ctl_expand_number(value, &uo);
2132         uss = us / cbe_lun->blocksize;
2133         uos = uo / cbe_lun->blocksize;
2134         if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
2135             ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
2136                 cbe_lun->ublockexp = fls(uss) - 1;
2137                 cbe_lun->ublockoff = (uss - uos) % uss;
2138         }
2139
2140         cbe_lun->atomicblock = atomic / cbe_lun->blocksize;
2141         cbe_lun->opttxferlen = maxio / cbe_lun->blocksize;
2142
2143         if (be_lun->dispatch == ctl_be_block_dispatch_zvol) {
2144                 unmap = 1;
2145         } else {
2146                 struct diocgattr_arg    arg;
2147
2148                 strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name));
2149                 arg.len = sizeof(arg.value.i);
2150                 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
2151                     curthread);
2152                 unmap = (error == 0) ? arg.value.i : 0;
2153         }
2154         value = dnvlist_get_string(cbe_lun->options, "unmap", NULL);
2155         if (value != NULL)
2156                 unmap = (strcmp(value, "on") == 0);
2157         if (unmap)
2158                 cbe_lun->flags |= CTL_LUN_FLAG_UNMAP;
2159         else
2160                 cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
2161
2162         dev_relthread(dev, ref);
2163         return (0);
2164 }
2165
2166 static int
2167 ctl_be_block_close(struct ctl_be_block_lun *be_lun)
2168 {
2169         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2170         int flags;
2171
2172         if (be_lun->vn) {
2173                 flags = FREAD;
2174                 if ((cbe_lun->flags & CTL_LUN_FLAG_READONLY) == 0)
2175                         flags |= FWRITE;
2176                 (void)vn_close(be_lun->vn, flags, NOCRED, curthread);
2177                 be_lun->vn = NULL;
2178
2179                 switch (be_lun->dev_type) {
2180                 case CTL_BE_BLOCK_DEV:
2181                         break;
2182                 case CTL_BE_BLOCK_FILE:
2183                         if (be_lun->backend.file.cred != NULL) {
2184                                 crfree(be_lun->backend.file.cred);
2185                                 be_lun->backend.file.cred = NULL;
2186                         }
2187                         break;
2188                 case CTL_BE_BLOCK_NONE:
2189                         break;
2190                 default:
2191                         panic("Unexpected backend type %d", be_lun->dev_type);
2192                         break;
2193                 }
2194                 be_lun->dev_type = CTL_BE_BLOCK_NONE;
2195         }
2196         return (0);
2197 }
2198
2199 static int
2200 ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
2201 {
2202         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2203         struct nameidata nd;
2204         const char      *value;
2205         int              error, flags;
2206
2207         error = 0;
2208         if (rootvnode == NULL) {
2209                 snprintf(req->error_str, sizeof(req->error_str),
2210                          "Root filesystem is not mounted");
2211                 return (1);
2212         }
2213         pwd_ensure_dirs();
2214
2215         value = dnvlist_get_string(cbe_lun->options, "file", NULL);
2216         if (value == NULL) {
2217                 snprintf(req->error_str, sizeof(req->error_str),
2218                          "no file argument specified");
2219                 return (1);
2220         }
2221         free(be_lun->dev_path, M_CTLBLK);
2222         be_lun->dev_path = strdup(value, M_CTLBLK);
2223
2224         flags = FREAD;
2225         value = dnvlist_get_string(cbe_lun->options, "readonly", NULL);
2226         if (value != NULL) {
2227                 if (strcmp(value, "on") != 0)
2228                         flags |= FWRITE;
2229         } else if (cbe_lun->lun_type == T_DIRECT)
2230                 flags |= FWRITE;
2231
2232 again:
2233         NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path);
2234         error = vn_open(&nd, &flags, 0, NULL);
2235         if ((error == EROFS || error == EACCES) && (flags & FWRITE)) {
2236                 flags &= ~FWRITE;
2237                 goto again;
2238         }
2239         if (error) {
2240                 /*
2241                  * This is the only reasonable guess we can make as far as
2242                  * path if the user doesn't give us a fully qualified path.
2243                  * If they want to specify a file, they need to specify the
2244                  * full path.
2245                  */
2246                 if (be_lun->dev_path[0] != '/') {
2247                         char *dev_name;
2248
2249                         asprintf(&dev_name, M_CTLBLK, "/dev/%s",
2250                                 be_lun->dev_path);
2251                         free(be_lun->dev_path, M_CTLBLK);
2252                         be_lun->dev_path = dev_name;
2253                         goto again;
2254                 }
2255                 snprintf(req->error_str, sizeof(req->error_str),
2256                     "error opening %s: %d", be_lun->dev_path, error);
2257                 return (error);
2258         }
2259         if (flags & FWRITE)
2260                 cbe_lun->flags &= ~CTL_LUN_FLAG_READONLY;
2261         else
2262                 cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
2263
2264         NDFREE_PNBUF(&nd);
2265         be_lun->vn = nd.ni_vp;
2266
2267         /* We only support disks and files. */
2268         if (vn_isdisk_error(be_lun->vn, &error)) {
2269                 error = ctl_be_block_open_dev(be_lun, req);
2270         } else if (be_lun->vn->v_type == VREG) {
2271                 error = ctl_be_block_open_file(be_lun, req);
2272         } else {
2273                 error = EINVAL;
2274                 snprintf(req->error_str, sizeof(req->error_str),
2275                          "%s is not a disk or plain file", be_lun->dev_path);
2276         }
2277         VOP_UNLOCK(be_lun->vn);
2278
2279         if (error != 0)
2280                 ctl_be_block_close(be_lun);
2281         cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
2282         if (be_lun->dispatch != ctl_be_block_dispatch_dev)
2283                 cbe_lun->serseq = CTL_LUN_SERSEQ_SOFT;
2284         value = dnvlist_get_string(cbe_lun->options, "serseq", NULL);
2285         if (value != NULL && strcmp(value, "on") == 0)
2286                 cbe_lun->serseq = CTL_LUN_SERSEQ_ON;
2287         else if (value != NULL && strcmp(value, "read") == 0)
2288                 cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
2289         else if (value != NULL && strcmp(value, "soft") == 0)
2290                 cbe_lun->serseq = CTL_LUN_SERSEQ_SOFT;
2291         else if (value != NULL && strcmp(value, "off") == 0)
2292                 cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
2293         return (0);
2294 }
2295
2296 static int
2297 ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2298 {
2299         struct ctl_be_lun *cbe_lun;
2300         struct ctl_be_block_lun *be_lun;
2301         struct ctl_lun_create_params *params;
2302         char num_thread_str[16];
2303         char tmpstr[32];
2304         const char *value;
2305         int retval, num_threads;
2306         int tmp_num_threads;
2307
2308         params = &req->reqdata.create;
2309         retval = 0;
2310         req->status = CTL_LUN_OK;
2311
2312         be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);
2313         cbe_lun = &be_lun->cbe_lun;
2314         be_lun->params = req->reqdata.create;
2315         be_lun->softc = softc;
2316         STAILQ_INIT(&be_lun->input_queue);
2317         STAILQ_INIT(&be_lun->config_read_queue);
2318         STAILQ_INIT(&be_lun->config_write_queue);
2319         STAILQ_INIT(&be_lun->datamove_queue);
2320         mtx_init(&be_lun->io_lock, "ctlblock io", NULL, MTX_DEF);
2321         mtx_init(&be_lun->queue_lock, "ctlblock queue", NULL, MTX_DEF);
2322         cbe_lun->options = nvlist_clone(req->args_nvl);
2323
2324         if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
2325                 cbe_lun->lun_type = params->device_type;
2326         else
2327                 cbe_lun->lun_type = T_DIRECT;
2328         be_lun->flags = 0;
2329         cbe_lun->flags = 0;
2330         value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL);
2331         if (value != NULL) {
2332                 if (strcmp(value, "primary") == 0)
2333                         cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2334         } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
2335                 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2336
2337         if (cbe_lun->lun_type == T_DIRECT ||
2338             cbe_lun->lun_type == T_CDROM) {
2339                 be_lun->size_bytes = params->lun_size_bytes;
2340                 if (params->blocksize_bytes != 0)
2341                         cbe_lun->blocksize = params->blocksize_bytes;
2342                 else if (cbe_lun->lun_type == T_CDROM)
2343                         cbe_lun->blocksize = 2048;
2344                 else
2345                         cbe_lun->blocksize = 512;
2346                 be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
2347                 cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
2348                     0 : (be_lun->size_blocks - 1);
2349
2350                 if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
2351                     control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
2352                         retval = ctl_be_block_open(be_lun, req);
2353                         if (retval != 0) {
2354                                 retval = 0;
2355                                 req->status = CTL_LUN_WARNING;
2356                         }
2357                 }
2358                 num_threads = cbb_num_threads;
2359         } else {
2360                 num_threads = 1;
2361         }
2362
2363         value = dnvlist_get_string(cbe_lun->options, "num_threads", NULL);
2364         if (value != NULL) {
2365                 tmp_num_threads = strtol(value, NULL, 0);
2366
2367                 /*
2368                  * We don't let the user specify less than one
2369                  * thread, but hope he's clueful enough not to
2370                  * specify 1000 threads.
2371                  */
2372                 if (tmp_num_threads < 1) {
2373                         snprintf(req->error_str, sizeof(req->error_str),
2374                                  "invalid number of threads %s",
2375                                  num_thread_str);
2376                         goto bailout_error;
2377                 }
2378                 num_threads = tmp_num_threads;
2379         }
2380
2381         if (be_lun->vn == NULL)
2382                 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2383         /* Tell the user the blocksize we ended up using */
2384         params->lun_size_bytes = be_lun->size_bytes;
2385         params->blocksize_bytes = cbe_lun->blocksize;
2386         if (params->flags & CTL_LUN_FLAG_ID_REQ) {
2387                 cbe_lun->req_lun_id = params->req_lun_id;
2388                 cbe_lun->flags |= CTL_LUN_FLAG_ID_REQ;
2389         } else
2390                 cbe_lun->req_lun_id = 0;
2391
2392         cbe_lun->lun_shutdown = ctl_be_block_lun_shutdown;
2393         cbe_lun->be = &ctl_be_block_driver;
2394
2395         if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) {
2396                 snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%04d",
2397                          softc->num_luns);
2398                 strncpy((char *)cbe_lun->serial_num, tmpstr,
2399                         MIN(sizeof(cbe_lun->serial_num), sizeof(tmpstr)));
2400
2401                 /* Tell the user what we used for a serial number */
2402                 strncpy((char *)params->serial_num, tmpstr,
2403                         MIN(sizeof(params->serial_num), sizeof(tmpstr)));
2404         } else {
2405                 strncpy((char *)cbe_lun->serial_num, params->serial_num,
2406                         MIN(sizeof(cbe_lun->serial_num),
2407                         sizeof(params->serial_num)));
2408         }
2409         if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) {
2410                 snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%04d", softc->num_luns);
2411                 strncpy((char *)cbe_lun->device_id, tmpstr,
2412                         MIN(sizeof(cbe_lun->device_id), sizeof(tmpstr)));
2413
2414                 /* Tell the user what we used for a device ID */
2415                 strncpy((char *)params->device_id, tmpstr,
2416                         MIN(sizeof(params->device_id), sizeof(tmpstr)));
2417         } else {
2418                 strncpy((char *)cbe_lun->device_id, params->device_id,
2419                         MIN(sizeof(cbe_lun->device_id),
2420                             sizeof(params->device_id)));
2421         }
2422
2423         TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);
2424
2425         be_lun->io_taskqueue = taskqueue_create("ctlblocktq", M_WAITOK,
2426             taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);
2427
2428         if (be_lun->io_taskqueue == NULL) {
2429                 snprintf(req->error_str, sizeof(req->error_str),
2430                          "unable to create taskqueue");
2431                 goto bailout_error;
2432         }
2433
2434         /*
2435          * Note that we start the same number of threads by default for
2436          * both the file case and the block device case.  For the file
2437          * case, we need multiple threads to allow concurrency, because the
2438          * vnode interface is designed to be a blocking interface.  For the
2439          * block device case, ZFS zvols at least will block the caller's
2440          * context in many instances, and so we need multiple threads to
2441          * overcome that problem.  Other block devices don't need as many
2442          * threads, but they shouldn't cause too many problems.
2443          *
2444          * If the user wants to just have a single thread for a block
2445          * device, he can specify that when the LUN is created, or change
2446          * the tunable/sysctl to alter the default number of threads.
2447          */
2448         retval = taskqueue_start_threads_in_proc(&be_lun->io_taskqueue,
2449                                          /*num threads*/num_threads,
2450                                          /*priority*/PUSER,
2451                                          /*proc*/control_softc->ctl_proc,
2452                                          /*thread name*/"block");
2453
2454         if (retval != 0)
2455                 goto bailout_error;
2456
2457         be_lun->num_threads = num_threads;
2458
2459         retval = ctl_add_lun(&be_lun->cbe_lun);
2460         if (retval != 0) {
2461                 snprintf(req->error_str, sizeof(req->error_str),
2462                          "ctl_add_lun() returned error %d, see dmesg for "
2463                          "details", retval);
2464                 retval = 0;
2465                 goto bailout_error;
2466         }
2467
2468         be_lun->disk_stats = devstat_new_entry("cbb", cbe_lun->lun_id,
2469                                                cbe_lun->blocksize,
2470                                                DEVSTAT_ALL_SUPPORTED,
2471                                                cbe_lun->lun_type
2472                                                | DEVSTAT_TYPE_IF_OTHER,
2473                                                DEVSTAT_PRIORITY_OTHER);
2474
2475         mtx_lock(&softc->lock);
2476         softc->num_luns++;
2477         SLIST_INSERT_HEAD(&softc->lun_list, be_lun, links);
2478         mtx_unlock(&softc->lock);
2479
2480         params->req_lun_id = cbe_lun->lun_id;
2481
2482         return (retval);
2483
2484 bailout_error:
2485         req->status = CTL_LUN_ERROR;
2486
2487         if (be_lun->io_taskqueue != NULL)
2488                 taskqueue_free(be_lun->io_taskqueue);
2489         ctl_be_block_close(be_lun);
2490         if (be_lun->dev_path != NULL)
2491                 free(be_lun->dev_path, M_CTLBLK);
2492         nvlist_destroy(cbe_lun->options);
2493         mtx_destroy(&be_lun->queue_lock);
2494         mtx_destroy(&be_lun->io_lock);
2495         free(be_lun, M_CTLBLK);
2496
2497         return (retval);
2498 }
2499
2500 static int
2501 ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2502 {
2503         struct ctl_lun_rm_params *params;
2504         struct ctl_be_block_lun *be_lun;
2505         struct ctl_be_lun *cbe_lun;
2506         int retval;
2507
2508         params = &req->reqdata.rm;
2509
2510         sx_xlock(&softc->modify_lock);
2511         mtx_lock(&softc->lock);
2512         SLIST_FOREACH(be_lun, &softc->lun_list, links) {
2513                 if (be_lun->cbe_lun.lun_id == params->lun_id) {
2514                         SLIST_REMOVE(&softc->lun_list, be_lun,
2515                             ctl_be_block_lun, links);
2516                         softc->num_luns--;
2517                         break;
2518                 }
2519         }
2520         mtx_unlock(&softc->lock);
2521         sx_xunlock(&softc->modify_lock);
2522         if (be_lun == NULL) {
2523                 snprintf(req->error_str, sizeof(req->error_str),
2524                          "LUN %u is not managed by the block backend",
2525                          params->lun_id);
2526                 goto bailout_error;
2527         }
2528         cbe_lun = &be_lun->cbe_lun;
2529
2530         if (be_lun->vn != NULL) {
2531                 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2532                 ctl_lun_no_media(cbe_lun);
2533                 taskqueue_drain_all(be_lun->io_taskqueue);
2534                 ctl_be_block_close(be_lun);
2535         }
2536
2537         mtx_lock(&softc->lock);
2538         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2539         mtx_unlock(&softc->lock);
2540
2541         retval = ctl_remove_lun(cbe_lun);
2542         if (retval != 0) {
2543                 snprintf(req->error_str, sizeof(req->error_str),
2544                          "error %d returned from ctl_remove_lun() for "
2545                          "LUN %d", retval, params->lun_id);
2546                 mtx_lock(&softc->lock);
2547                 be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2548                 mtx_unlock(&softc->lock);
2549                 goto bailout_error;
2550         }
2551
2552         mtx_lock(&softc->lock);
2553         while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2554                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblockrm", 0);
2555                 if (retval == EINTR)
2556                         break;
2557         }
2558         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2559         if (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {
2560                 mtx_unlock(&softc->lock);
2561                 free(be_lun, M_CTLBLK);
2562         } else {
2563                 mtx_unlock(&softc->lock);
2564                 return (EINTR);
2565         }
2566
2567         req->status = CTL_LUN_OK;
2568         return (0);
2569
2570 bailout_error:
2571         req->status = CTL_LUN_ERROR;
2572         return (0);
2573 }
2574
2575 static int
2576 ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2577 {
2578         struct ctl_lun_modify_params *params;
2579         struct ctl_be_block_lun *be_lun;
2580         struct ctl_be_lun *cbe_lun;
2581         const char *value;
2582         uint64_t oldsize;
2583         int error, wasprim;
2584
2585         params = &req->reqdata.modify;
2586
2587         sx_xlock(&softc->modify_lock);
2588         mtx_lock(&softc->lock);
2589         SLIST_FOREACH(be_lun, &softc->lun_list, links) {
2590                 if (be_lun->cbe_lun.lun_id == params->lun_id)
2591                         break;
2592         }
2593         mtx_unlock(&softc->lock);
2594         if (be_lun == NULL) {
2595                 snprintf(req->error_str, sizeof(req->error_str),
2596                          "LUN %u is not managed by the block backend",
2597                          params->lun_id);
2598                 goto bailout_error;
2599         }
2600         cbe_lun = &be_lun->cbe_lun;
2601
2602         if (params->lun_size_bytes != 0)
2603                 be_lun->params.lun_size_bytes = params->lun_size_bytes;
2604
2605         if (req->args_nvl != NULL) {
2606                 nvlist_destroy(cbe_lun->options);
2607                 cbe_lun->options = nvlist_clone(req->args_nvl);
2608         }
2609
2610         wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY);
2611         value = dnvlist_get_string(cbe_lun->options, "ha_role", NULL);
2612         if (value != NULL) {
2613                 if (strcmp(value, "primary") == 0)
2614                         cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2615                 else
2616                         cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
2617         } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
2618                 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2619         else
2620                 cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
2621         if (wasprim != (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)) {
2622                 if (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)
2623                         ctl_lun_primary(cbe_lun);
2624                 else
2625                         ctl_lun_secondary(cbe_lun);
2626         }
2627
2628         oldsize = be_lun->size_blocks;
2629         if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
2630             control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
2631                 if (be_lun->vn == NULL)
2632                         error = ctl_be_block_open(be_lun, req);
2633                 else if (vn_isdisk_error(be_lun->vn, &error))
2634                         error = ctl_be_block_open_dev(be_lun, req);
2635                 else if (be_lun->vn->v_type == VREG) {
2636                         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
2637                         error = ctl_be_block_open_file(be_lun, req);
2638                         VOP_UNLOCK(be_lun->vn);
2639                 } else
2640                         error = EINVAL;
2641                 if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) &&
2642                     be_lun->vn != NULL) {
2643                         cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
2644                         ctl_lun_has_media(cbe_lun);
2645                 } else if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) == 0 &&
2646                     be_lun->vn == NULL) {
2647                         cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2648                         ctl_lun_no_media(cbe_lun);
2649                 }
2650                 cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
2651         } else {
2652                 if (be_lun->vn != NULL) {
2653                         cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2654                         ctl_lun_no_media(cbe_lun);
2655                         taskqueue_drain_all(be_lun->io_taskqueue);
2656                         error = ctl_be_block_close(be_lun);
2657                 } else
2658                         error = 0;
2659         }
2660         if (be_lun->size_blocks != oldsize)
2661                 ctl_lun_capacity_changed(cbe_lun);
2662
2663         /* Tell the user the exact size we ended up using */
2664         params->lun_size_bytes = be_lun->size_bytes;
2665
2666         sx_xunlock(&softc->modify_lock);
2667         req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK;
2668         return (0);
2669
2670 bailout_error:
2671         sx_xunlock(&softc->modify_lock);
2672         req->status = CTL_LUN_ERROR;
2673         return (0);
2674 }
2675
2676 static void
2677 ctl_be_block_lun_shutdown(struct ctl_be_lun *cbe_lun)
2678 {
2679         struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)cbe_lun;
2680         struct ctl_be_block_softc *softc = be_lun->softc;
2681
2682         taskqueue_drain_all(be_lun->io_taskqueue);
2683         taskqueue_free(be_lun->io_taskqueue);
2684         if (be_lun->disk_stats != NULL)
2685                 devstat_remove_entry(be_lun->disk_stats);
2686         nvlist_destroy(be_lun->cbe_lun.options);
2687         free(be_lun->dev_path, M_CTLBLK);
2688         mtx_destroy(&be_lun->queue_lock);
2689         mtx_destroy(&be_lun->io_lock);
2690
2691         mtx_lock(&softc->lock);
2692         be_lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;
2693         if (be_lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2694                 wakeup(be_lun);
2695         else
2696                 free(be_lun, M_CTLBLK);
2697         mtx_unlock(&softc->lock);
2698 }
2699
2700 static int
2701 ctl_be_block_config_write(union ctl_io *io)
2702 {
2703         struct ctl_be_block_lun *be_lun;
2704         struct ctl_be_lun *cbe_lun;
2705         int retval;
2706
2707         DPRINTF("entered\n");
2708
2709         cbe_lun = CTL_BACKEND_LUN(io);
2710         be_lun = (struct ctl_be_block_lun *)cbe_lun;
2711
2712         retval = 0;
2713         switch (io->scsiio.cdb[0]) {
2714         case SYNCHRONIZE_CACHE:
2715         case SYNCHRONIZE_CACHE_16:
2716         case WRITE_SAME_10:
2717         case WRITE_SAME_16:
2718         case UNMAP:
2719                 /*
2720                  * The upper level CTL code will filter out any CDBs with
2721                  * the immediate bit set and return the proper error.
2722                  *
2723                  * We don't really need to worry about what LBA range the
2724                  * user asked to be synced out.  When they issue a sync
2725                  * cache command, we'll sync out the whole thing.
2726                  */
2727                 mtx_lock(&be_lun->queue_lock);
2728                 STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,
2729                                    links);
2730                 mtx_unlock(&be_lun->queue_lock);
2731                 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
2732                 break;
2733         case START_STOP_UNIT: {
2734                 struct scsi_start_stop_unit *cdb;
2735                 struct ctl_lun_req req;
2736
2737                 cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
2738                 if ((cdb->how & SSS_PC_MASK) != 0) {
2739                         ctl_set_success(&io->scsiio);
2740                         ctl_config_write_done(io);
2741                         break;
2742                 }
2743                 if (cdb->how & SSS_START) {
2744                         if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) {
2745                                 retval = ctl_be_block_open(be_lun, &req);
2746                                 cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
2747                                 if (retval == 0) {
2748                                         cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
2749                                         ctl_lun_has_media(cbe_lun);
2750                                 } else {
2751                                         cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2752                                         ctl_lun_no_media(cbe_lun);
2753                                 }
2754                         }
2755                         ctl_start_lun(cbe_lun);
2756                 } else {
2757                         ctl_stop_lun(cbe_lun);
2758                         if (cdb->how & SSS_LOEJ) {
2759                                 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2760                                 cbe_lun->flags |= CTL_LUN_FLAG_EJECTED;
2761                                 ctl_lun_ejected(cbe_lun);
2762                                 if (be_lun->vn != NULL)
2763                                         ctl_be_block_close(be_lun);
2764                         }
2765                 }
2766
2767                 ctl_set_success(&io->scsiio);
2768                 ctl_config_write_done(io);
2769                 break;
2770         }
2771         case PREVENT_ALLOW:
2772                 ctl_set_success(&io->scsiio);
2773                 ctl_config_write_done(io);
2774                 break;
2775         default:
2776                 ctl_set_invalid_opcode(&io->scsiio);
2777                 ctl_config_write_done(io);
2778                 retval = CTL_RETVAL_COMPLETE;
2779                 break;
2780         }
2781
2782         return (retval);
2783 }
2784
2785 static int
2786 ctl_be_block_config_read(union ctl_io *io)
2787 {
2788         struct ctl_be_block_lun *be_lun;
2789         int retval = 0;
2790
2791         DPRINTF("entered\n");
2792
2793         be_lun = (struct ctl_be_block_lun *)CTL_BACKEND_LUN(io);
2794
2795         switch (io->scsiio.cdb[0]) {
2796         case SERVICE_ACTION_IN:
2797                 if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) {
2798                         mtx_lock(&be_lun->queue_lock);
2799                         STAILQ_INSERT_TAIL(&be_lun->config_read_queue,
2800                             &io->io_hdr, links);
2801                         mtx_unlock(&be_lun->queue_lock);
2802                         taskqueue_enqueue(be_lun->io_taskqueue,
2803                             &be_lun->io_task);
2804                         retval = CTL_RETVAL_QUEUED;
2805                         break;
2806                 }
2807                 ctl_set_invalid_field(&io->scsiio,
2808                                       /*sks_valid*/ 1,
2809                                       /*command*/ 1,
2810                                       /*field*/ 1,
2811                                       /*bit_valid*/ 1,
2812                                       /*bit*/ 4);
2813                 ctl_config_read_done(io);
2814                 retval = CTL_RETVAL_COMPLETE;
2815                 break;
2816         default:
2817                 ctl_set_invalid_opcode(&io->scsiio);
2818                 ctl_config_read_done(io);
2819                 retval = CTL_RETVAL_COMPLETE;
2820                 break;
2821         }
2822
2823         return (retval);
2824 }
2825
2826 static int
2827 ctl_be_block_lun_info(struct ctl_be_lun *cbe_lun, struct sbuf *sb)
2828 {
2829         struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)cbe_lun;
2830         int retval;
2831
2832         retval = sbuf_printf(sb, "\t<num_threads>");
2833         if (retval != 0)
2834                 goto bailout;
2835         retval = sbuf_printf(sb, "%d", lun->num_threads);
2836         if (retval != 0)
2837                 goto bailout;
2838         retval = sbuf_printf(sb, "</num_threads>\n");
2839
2840 bailout:
2841         return (retval);
2842 }
2843
2844 static uint64_t
2845 ctl_be_block_lun_attr(struct ctl_be_lun *cbe_lun, const char *attrname)
2846 {
2847         struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)cbe_lun;
2848
2849         if (lun->getattr == NULL)
2850                 return (UINT64_MAX);
2851         return (lun->getattr(lun, attrname));
2852 }
2853
2854 static int
2855 ctl_be_block_init(void)
2856 {
2857         struct ctl_be_block_softc *softc = &backend_block_softc;
2858
2859         sx_init(&softc->modify_lock, "ctlblock modify");
2860         mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF);
2861         softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io),
2862             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
2863         softc->bufmin_zone = uma_zcreate("ctlblockmin", CTLBLK_MIN_SEG,
2864             NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
2865         if (CTLBLK_MIN_SEG < CTLBLK_MAX_SEG)
2866                 softc->bufmax_zone = uma_zcreate("ctlblockmax", CTLBLK_MAX_SEG,
2867                     NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
2868         SLIST_INIT(&softc->lun_list);
2869         return (0);
2870 }
2871
2872 static int
2873 ctl_be_block_shutdown(void)
2874 {
2875         struct ctl_be_block_softc *softc = &backend_block_softc;
2876         struct ctl_be_block_lun *lun;
2877
2878         mtx_lock(&softc->lock);
2879         while ((lun = SLIST_FIRST(&softc->lun_list)) != NULL) {
2880                 SLIST_REMOVE_HEAD(&softc->lun_list, links);
2881                 softc->num_luns--;
2882                 /*
2883                  * Drop our lock here.  Since ctl_remove_lun() can call
2884                  * back into us, this could potentially lead to a recursive
2885                  * lock of the same mutex, which would cause a hang.
2886                  */
2887                 mtx_unlock(&softc->lock);
2888                 ctl_remove_lun(&lun->cbe_lun);
2889                 mtx_lock(&softc->lock);
2890         }
2891         mtx_unlock(&softc->lock);
2892         uma_zdestroy(softc->bufmin_zone);
2893         if (CTLBLK_MIN_SEG < CTLBLK_MAX_SEG)
2894                 uma_zdestroy(softc->bufmax_zone);
2895         uma_zdestroy(softc->beio_zone);
2896         mtx_destroy(&softc->lock);
2897         sx_destroy(&softc->modify_lock);
2898         return (0);
2899 }