sys/cam/ctl/ctl_backend_block.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003 Silicon Graphics International Corp.
   5  * Copyright (c) 2009-2011 Spectra Logic Corporation
   6  * Copyright (c) 2012 The FreeBSD Foundation
   7  * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
   8  * All rights reserved.
   9  *
  10  * Portions of this software were developed by Edward Tomasz Napierala
  11  * under sponsorship from the FreeBSD Foundation.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions, and the following disclaimer,
  18  *    without modification.
  19  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  20  *    substantially similar to the "NO WARRANTY" disclaimer below
  21  *    ("Disclaimer") and any redistribution must be conditioned upon
  22  *    including a substantially similar Disclaimer requirement for further
  23  *    binary redistribution.
  24  *
  25  * NO WARRANTY
  26  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  27  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  28  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  29  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  30  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  34  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  35  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  36  * POSSIBILITY OF SUCH DAMAGES.
  37  *
  38  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
  39  */
  40 /*
  41  * CAM Target Layer driver backend for block devices.
  42  *
  43  * Author: Ken Merry <ken@FreeBSD.org>
  44  */
  45 #include <sys/cdefs.h>
  46 __FBSDID("$FreeBSD$");
  47
  48 #include <sys/param.h>
  49 #include <sys/systm.h>
  50 #include <sys/kernel.h>
  51 #include <sys/types.h>
  52 #include <sys/kthread.h>
  53 #include <sys/bio.h>
  54 #include <sys/fcntl.h>
  55 #include <sys/limits.h>
  56 #include <sys/lock.h>
  57 #include <sys/mutex.h>
  58 #include <sys/condvar.h>
  59 #include <sys/malloc.h>
  60 #include <sys/conf.h>
  61 #include <sys/ioccom.h>
  62 #include <sys/queue.h>
  63 #include <sys/sbuf.h>
  64 #include <sys/endian.h>
  65 #include <sys/uio.h>
  66 #include <sys/buf.h>
  67 #include <sys/taskqueue.h>
  68 #include <sys/vnode.h>
  69 #include <sys/namei.h>
  70 #include <sys/mount.h>
  71 #include <sys/disk.h>
  72 #include <sys/fcntl.h>
  73 #include <sys/filedesc.h>
  74 #include <sys/filio.h>
  75 #include <sys/proc.h>
  76 #include <sys/pcpu.h>
  77 #include <sys/module.h>
  78 #include <sys/sdt.h>
  79 #include <sys/devicestat.h>
  80 #include <sys/sysctl.h>
  81
  82 #include <geom/geom.h>
  83
  84 #include <cam/cam.h>
  85 #include <cam/scsi/scsi_all.h>
  86 #include <cam/scsi/scsi_da.h>
  87 #include <cam/ctl/ctl_io.h>
  88 #include <cam/ctl/ctl.h>
  89 #include <cam/ctl/ctl_backend.h>
  90 #include <cam/ctl/ctl_ioctl.h>
  91 #include <cam/ctl/ctl_ha.h>
  92 #include <cam/ctl/ctl_scsi_all.h>
  93 #include <cam/ctl/ctl_private.h>
  94 #include <cam/ctl/ctl_error.h>
  95
  96 /*
  97  * The idea here is that we'll allocate enough S/G space to hold a 1MB
  98  * I/O.  If we get an I/O larger than that, we'll split it.
  99  */
 100 #define CTLBLK_HALF_IO_SIZE     (512 * 1024)
 101 #define CTLBLK_MAX_IO_SIZE      (CTLBLK_HALF_IO_SIZE * 2)
 102 #define CTLBLK_MAX_SEG          MAXPHYS
 103 #define CTLBLK_HALF_SEGS        MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1)
 104 #define CTLBLK_MAX_SEGS         (CTLBLK_HALF_SEGS * 2)
 105
 106 #ifdef CTLBLK_DEBUG
 107 #define DPRINTF(fmt, args...) \
 108     printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args)
 109 #else
 110 #define DPRINTF(fmt, args...) do {} while(0)
 111 #endif
 112
 113 #define PRIV(io)        \
 114     ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND])
 115 #define ARGS(io)        \
 116     ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN])
 117
 118 SDT_PROVIDER_DEFINE(cbb);
 119
 120 typedef enum {
 121         CTL_BE_BLOCK_LUN_UNCONFIGURED   = 0x01,
 122         CTL_BE_BLOCK_LUN_CONFIG_ERR     = 0x02,
 123         CTL_BE_BLOCK_LUN_WAITING        = 0x04,
 124 } ctl_be_block_lun_flags;
 125
 126 typedef enum {
 127         CTL_BE_BLOCK_NONE,
 128         CTL_BE_BLOCK_DEV,
 129         CTL_BE_BLOCK_FILE
 130 } ctl_be_block_type;
 131
 132 struct ctl_be_block_filedata {
 133         struct ucred *cred;
 134 };
 135
 136 union ctl_be_block_bedata {
 137         struct ctl_be_block_filedata file;
 138 };
 139
 140 struct ctl_be_block_io;
 141 struct ctl_be_block_lun;
 142
 143 typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
 144                                struct ctl_be_block_io *beio);
 145 typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun,
 146                                   const char *attrname);
 147
 148 /*
 149  * Backend LUN structure.  There is a 1:1 mapping between a block device
 150  * and a backend block LUN, and between a backend block LUN and a CTL LUN.
 151  */
 152 struct ctl_be_block_lun {
 153         struct ctl_lun_create_params params;
 154         char lunname[32];
 155         char *dev_path;
 156         ctl_be_block_type dev_type;
 157         struct vnode *vn;
 158         union ctl_be_block_bedata backend;
 159         cbb_dispatch_t dispatch;
 160         cbb_dispatch_t lun_flush;
 161         cbb_dispatch_t unmap;
 162         cbb_dispatch_t get_lba_status;
 163         cbb_getattr_t getattr;
 164         uma_zone_t lun_zone;
 165         uint64_t size_blocks;
 166         uint64_t size_bytes;
 167         struct ctl_be_block_softc *softc;
 168         struct devstat *disk_stats;
 169         ctl_be_block_lun_flags flags;
 170         STAILQ_ENTRY(ctl_be_block_lun) links;
 171         struct ctl_be_lun cbe_lun;
 172         struct taskqueue *io_taskqueue;
 173         struct task io_task;
 174         int num_threads;
 175         STAILQ_HEAD(, ctl_io_hdr) input_queue;
 176         STAILQ_HEAD(, ctl_io_hdr) config_read_queue;
 177         STAILQ_HEAD(, ctl_io_hdr) config_write_queue;
 178         STAILQ_HEAD(, ctl_io_hdr) datamove_queue;
 179         struct mtx_padalign io_lock;
 180         struct mtx_padalign queue_lock;
 181 };
 182
 183 /*
 184  * Overall softc structure for the block backend module.
 185  */
 186 struct ctl_be_block_softc {
 187         struct mtx                       lock;
 188         uma_zone_t                       beio_zone;
 189         int                              num_luns;
 190         STAILQ_HEAD(, ctl_be_block_lun)  lun_list;
 191 };
 192
 193 static struct ctl_be_block_softc backend_block_softc;
 194
 195 /*
 196  * Per-I/O information.
 197  */
 198 struct ctl_be_block_io {
 199         union ctl_io                    *io;
 200         struct ctl_sg_entry             sg_segs[CTLBLK_MAX_SEGS];
 201         struct iovec                    xiovecs[CTLBLK_MAX_SEGS];
 202         int                             bio_cmd;
 203         int                             num_segs;
 204         int                             num_bios_sent;
 205         int                             num_bios_done;
 206         int                             send_complete;
 207         int                             first_error;
 208         uint64_t                        first_error_offset;
 209         struct bintime                  ds_t0;
 210         devstat_tag_type                ds_tag_type;
 211         devstat_trans_flags             ds_trans_type;
 212         uint64_t                        io_len;
 213         uint64_t                        io_offset;
 214         int                             io_arg;
 215         struct ctl_be_block_softc       *softc;
 216         struct ctl_be_block_lun         *lun;
 217         void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */
 218 };
 219
 220 extern struct ctl_softc *control_softc;
 221
 222 static int cbb_num_threads = 14;
 223 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0,
 224             "CAM Target Layer Block Backend");
 225 SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN,
 226            &cbb_num_threads, 0, "Number of threads per backing file");
 227
 228 static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
 229 static void ctl_free_beio(struct ctl_be_block_io *beio);
 230 static void ctl_complete_beio(struct ctl_be_block_io *beio);
 231 static int ctl_be_block_move_done(union ctl_io *io);
 232 static void ctl_be_block_biodone(struct bio *bio);
 233 static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 234                                     struct ctl_be_block_io *beio);
 235 static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 236                                        struct ctl_be_block_io *beio);
 237 static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
 238                                   struct ctl_be_block_io *beio);
 239 static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun,
 240                                          const char *attrname);
 241 static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 242                                    struct ctl_be_block_io *beio);
 243 static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
 244                                    struct ctl_be_block_io *beio);
 245 static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
 246                                       struct ctl_be_block_io *beio);
 247 static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun,
 248                                          const char *attrname);
 249 static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
 250                                     union ctl_io *io);
 251 static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
 252                                     union ctl_io *io);
 253 static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 254                                   union ctl_io *io);
 255 static void ctl_be_block_worker(void *context, int pending);
 256 static int ctl_be_block_submit(union ctl_io *io);
 257 static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
 258                                    int flag, struct thread *td);
 259 static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
 260                                   struct ctl_lun_req *req);
 261 static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
 262                                  struct ctl_lun_req *req);
 263 static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
 264 static int ctl_be_block_open(struct ctl_be_block_lun *be_lun,
 265                              struct ctl_lun_req *req);
 266 static int ctl_be_block_create(struct ctl_be_block_softc *softc,
 267                                struct ctl_lun_req *req);
 268 static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
 269                            struct ctl_lun_req *req);
 270 static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
 271                            struct ctl_lun_req *req);
 272 static void ctl_be_block_lun_shutdown(void *be_lun);
 273 static void ctl_be_block_lun_config_status(void *be_lun,
 274                                            ctl_lun_config_status status);
 275 static int ctl_be_block_config_write(union ctl_io *io);
 276 static int ctl_be_block_config_read(union ctl_io *io);
 277 static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb);
 278 static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname);
 279 static int ctl_be_block_init(void);
 280 static int ctl_be_block_shutdown(void);
 281
 282 static struct ctl_backend_driver ctl_be_block_driver =
 283 {
 284         .name = "block",
 285         .flags = CTL_BE_FLAG_HAS_CONFIG,
 286         .init = ctl_be_block_init,
 287         .shutdown = ctl_be_block_shutdown,
 288         .data_submit = ctl_be_block_submit,
 289         .data_move_done = ctl_be_block_move_done,
 290         .config_read = ctl_be_block_config_read,
 291         .config_write = ctl_be_block_config_write,
 292         .ioctl = ctl_be_block_ioctl,
 293         .lun_info = ctl_be_block_lun_info,
 294         .lun_attr = ctl_be_block_lun_attr
 295 };
 296
 297 MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend");
 298 CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
 299
 300 static struct ctl_be_block_io *
 301 ctl_alloc_beio(struct ctl_be_block_softc *softc)
 302 {
 303         struct ctl_be_block_io *beio;
 304
 305         beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO);
 306         beio->softc = softc;
 307         return (beio);
 308 }
 309
 310 static void
 311 ctl_free_beio(struct ctl_be_block_io *beio)
 312 {
 313         int duplicate_free;
 314         int i;
 315
 316         duplicate_free = 0;
 317
 318         for (i = 0; i < beio->num_segs; i++) {
 319                 if (beio->sg_segs[i].addr == NULL)
 320                         duplicate_free++;
 321
 322                 uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr);
 323                 beio->sg_segs[i].addr = NULL;
 324
 325                 /* For compare we had two equal S/G lists. */
 326                 if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) {
 327                         uma_zfree(beio->lun->lun_zone,
 328                             beio->sg_segs[i + CTLBLK_HALF_SEGS].addr);
 329                         beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL;
 330                 }
 331         }
 332
 333         if (duplicate_free > 0) {
 334                 printf("%s: %d duplicate frees out of %d segments\n", __func__,
 335                        duplicate_free, beio->num_segs);
 336         }
 337
 338         uma_zfree(beio->softc->beio_zone, beio);
 339 }
 340
 341 static void
 342 ctl_complete_beio(struct ctl_be_block_io *beio)
 343 {
 344         union ctl_io *io = beio->io;
 345
 346         if (beio->beio_cont != NULL) {
 347                 beio->beio_cont(beio);
 348         } else {
 349                 ctl_free_beio(beio);
 350                 ctl_data_submit_done(io);
 351         }
 352 }
 353
 354 static size_t
 355 cmp(uint8_t *a, uint8_t *b, size_t size)
 356 {
 357         size_t i;
 358
 359         for (i = 0; i < size; i++) {
 360                 if (a[i] != b[i])
 361                         break;
 362         }
 363         return (i);
 364 }
 365
 366 static void
 367 ctl_be_block_compare(union ctl_io *io)
 368 {
 369         struct ctl_be_block_io *beio;
 370         uint64_t off, res;
 371         int i;
 372         uint8_t info[8];
 373
 374         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 375         off = 0;
 376         for (i = 0; i < beio->num_segs; i++) {
 377                 res = cmp(beio->sg_segs[i].addr,
 378                     beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
 379                     beio->sg_segs[i].len);
 380                 off += res;
 381                 if (res < beio->sg_segs[i].len)
 382                         break;
 383         }
 384         if (i < beio->num_segs) {
 385                 scsi_u64to8b(off, info);
 386                 ctl_set_sense(&io->scsiio, /*current_error*/ 1,
 387                     /*sense_key*/ SSD_KEY_MISCOMPARE,
 388                     /*asc*/ 0x1D, /*ascq*/ 0x00,
 389                     /*type*/ SSD_ELEM_INFO,
 390                     /*size*/ sizeof(info), /*data*/ &info,
 391                     /*type*/ SSD_ELEM_NONE);
 392         } else
 393                 ctl_set_success(&io->scsiio);
 394 }
 395
 396 static int
 397 ctl_be_block_move_done(union ctl_io *io)
 398 {
 399         struct ctl_be_block_io *beio;
 400         struct ctl_be_block_lun *be_lun;
 401         struct ctl_lba_len_flags *lbalen;
 402 #ifdef CTL_TIME_IO
 403         struct bintime cur_bt;
 404 #endif
 405
 406         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 407         be_lun = beio->lun;
 408
 409         DPRINTF("entered\n");
 410
 411 #ifdef CTL_TIME_IO
 412         getbinuptime(&cur_bt);
 413         bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
 414         bintime_add(&io->io_hdr.dma_bt, &cur_bt);
 415 #endif
 416         io->io_hdr.num_dmas++;
 417         io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 418
 419         /*
 420          * We set status at this point for read commands, and write
 421          * commands with errors.
 422          */
 423         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
 424                 ;
 425         } else if ((io->io_hdr.port_status != 0) &&
 426             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
 427              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
 428                 ctl_set_internal_failure(&io->scsiio, /*sks_valid*/ 1,
 429                     /*retry_count*/ io->io_hdr.port_status);
 430         } else if (io->scsiio.kern_data_resid != 0 &&
 431             (io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_OUT &&
 432             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
 433              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
 434                 ctl_set_invalid_field_ciu(&io->scsiio);
 435         } else if ((io->io_hdr.port_status == 0) &&
 436             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
 437                 lbalen = ARGS(beio->io);
 438                 if (lbalen->flags & CTL_LLF_READ) {
 439                         ctl_set_success(&io->scsiio);
 440                 } else if (lbalen->flags & CTL_LLF_COMPARE) {
 441                         /* We have two data blocks ready for comparison. */
 442                         ctl_be_block_compare(io);
 443                 }
 444         }
 445
 446         /*
 447          * If this is a read, or a write with errors, it is done.
 448          */
 449         if ((beio->bio_cmd == BIO_READ)
 450          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)
 451          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) {
 452                 ctl_complete_beio(beio);
 453                 return (0);
 454         }
 455
 456         /*
 457          * At this point, we have a write and the DMA completed
 458          * successfully.  We now have to queue it to the task queue to
 459          * execute the backend I/O.  That is because we do blocking
 460          * memory allocations, and in the file backing case, blocking I/O.
 461          * This move done routine is generally called in the SIM's
 462          * interrupt context, and therefore we cannot block.
 463          */
 464         mtx_lock(&be_lun->queue_lock);
 465         STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
 466         mtx_unlock(&be_lun->queue_lock);
 467         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
 468
 469         return (0);
 470 }
 471
 472 static void
 473 ctl_be_block_biodone(struct bio *bio)
 474 {
 475         struct ctl_be_block_io *beio;
 476         struct ctl_be_block_lun *be_lun;
 477         union ctl_io *io;
 478         int error;
 479
 480         beio = bio->bio_caller1;
 481         be_lun = beio->lun;
 482         io = beio->io;
 483
 484         DPRINTF("entered\n");
 485
 486         error = bio->bio_error;
 487         mtx_lock(&be_lun->io_lock);
 488         if (error != 0 &&
 489             (beio->first_error == 0 ||
 490              bio->bio_offset < beio->first_error_offset)) {
 491                 beio->first_error = error;
 492                 beio->first_error_offset = bio->bio_offset;
 493         }
 494
 495         beio->num_bios_done++;
 496
 497         /*
 498          * XXX KDM will this cause WITNESS to complain?  Holding a lock
 499          * during the free might cause it to complain.
 500          */
 501         g_destroy_bio(bio);
 502
 503         /*
 504          * If the send complete bit isn't set, or we aren't the last I/O to
 505          * complete, then we're done.
 506          */
 507         if ((beio->send_complete == 0)
 508          || (beio->num_bios_done < beio->num_bios_sent)) {
 509                 mtx_unlock(&be_lun->io_lock);
 510                 return;
 511         }
 512
 513         /*
 514          * At this point, we've verified that we are the last I/O to
 515          * complete, so it's safe to drop the lock.
 516          */
 517         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 518             beio->ds_tag_type, beio->ds_trans_type,
 519             /*now*/ NULL, /*then*/&beio->ds_t0);
 520         mtx_unlock(&be_lun->io_lock);
 521
 522         /*
 523          * If there are any errors from the backing device, we fail the
 524          * entire I/O with a medium error.
 525          */
 526         error = beio->first_error;
 527         if (error != 0) {
 528                 if (error == EOPNOTSUPP) {
 529                         ctl_set_invalid_opcode(&io->scsiio);
 530                 } else if (error == ENOSPC || error == EDQUOT) {
 531                         ctl_set_space_alloc_fail(&io->scsiio);
 532                 } else if (error == EROFS || error == EACCES) {
 533                         ctl_set_hw_write_protected(&io->scsiio);
 534                 } else if (beio->bio_cmd == BIO_FLUSH) {
 535                         /* XXX KDM is there is a better error here? */
 536                         ctl_set_internal_failure(&io->scsiio,
 537                                                  /*sks_valid*/ 1,
 538                                                  /*retry_count*/ 0xbad2);
 539                 } else {
 540                         ctl_set_medium_error(&io->scsiio,
 541                             beio->bio_cmd == BIO_READ);
 542                 }
 543                 ctl_complete_beio(beio);
 544                 return;
 545         }
 546
 547         /*
 548          * If this is a write, a flush, a delete or verify, we're all done.
 549          * If this is a read, we can now send the data to the user.
 550          */
 551         if ((beio->bio_cmd == BIO_WRITE)
 552          || (beio->bio_cmd == BIO_FLUSH)
 553          || (beio->bio_cmd == BIO_DELETE)
 554          || (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 555                 ctl_set_success(&io->scsiio);
 556                 ctl_complete_beio(beio);
 557         } else {
 558                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 559                     beio->beio_cont == NULL) {
 560                         ctl_set_success(&io->scsiio);
 561                         ctl_serseq_done(io);
 562                 }
 563 #ifdef CTL_TIME_IO
 564                 getbinuptime(&io->io_hdr.dma_start_bt);
 565 #endif
 566                 ctl_datamove(io);
 567         }
 568 }
 569
 570 static void
 571 ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 572                         struct ctl_be_block_io *beio)
 573 {
 574         union ctl_io *io = beio->io;
 575         struct mount *mountpoint;
 576         int error, lock_flags;
 577
 578         DPRINTF("entered\n");
 579
 580         binuptime(&beio->ds_t0);
 581         mtx_lock(&be_lun->io_lock);
 582         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 583         mtx_unlock(&be_lun->io_lock);
 584
 585         (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 586
 587         if (MNT_SHARED_WRITES(mountpoint) ||
 588             ((mountpoint == NULL) && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 589                 lock_flags = LK_SHARED;
 590         else
 591                 lock_flags = LK_EXCLUSIVE;
 592         vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 593         error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT,
 594             curthread);
 595         VOP_UNLOCK(be_lun->vn, 0);
 596
 597         vn_finished_write(mountpoint);
 598
 599         mtx_lock(&be_lun->io_lock);
 600         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 601             beio->ds_tag_type, beio->ds_trans_type,
 602             /*now*/ NULL, /*then*/&beio->ds_t0);
 603         mtx_unlock(&be_lun->io_lock);
 604
 605         if (error == 0)
 606                 ctl_set_success(&io->scsiio);
 607         else {
 608                 /* XXX KDM is there is a better error here? */
 609                 ctl_set_internal_failure(&io->scsiio,
 610                                          /*sks_valid*/ 1,
 611                                          /*retry_count*/ 0xbad1);
 612         }
 613
 614         ctl_complete_beio(beio);
 615 }
 616
 617 SDT_PROBE_DEFINE1(cbb, , read, file_start, "uint64_t");
 618 SDT_PROBE_DEFINE1(cbb, , write, file_start, "uint64_t");
 619 SDT_PROBE_DEFINE1(cbb, , read, file_done,"uint64_t");
 620 SDT_PROBE_DEFINE1(cbb, , write, file_done, "uint64_t");
 621
 622 static void
 623 ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 624                            struct ctl_be_block_io *beio)
 625 {
 626         struct ctl_be_block_filedata *file_data;
 627         union ctl_io *io;
 628         struct uio xuio;
 629         struct iovec *xiovec;
 630         size_t s;
 631         int error, flags, i;
 632
 633         DPRINTF("entered\n");
 634
 635         file_data = &be_lun->backend.file;
 636         io = beio->io;
 637         flags = 0;
 638         if (ARGS(io)->flags & CTL_LLF_DPO)
 639                 flags |= IO_DIRECT;
 640         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 641                 flags |= IO_SYNC;
 642
 643         bzero(&xuio, sizeof(xuio));
 644         if (beio->bio_cmd == BIO_READ) {
 645                 SDT_PROBE0(cbb, , read, file_start);
 646                 xuio.uio_rw = UIO_READ;
 647         } else {
 648                 SDT_PROBE0(cbb, , write, file_start);
 649                 xuio.uio_rw = UIO_WRITE;
 650         }
 651         xuio.uio_offset = beio->io_offset;
 652         xuio.uio_resid = beio->io_len;
 653         xuio.uio_segflg = UIO_SYSSPACE;
 654         xuio.uio_iov = beio->xiovecs;
 655         xuio.uio_iovcnt = beio->num_segs;
 656         xuio.uio_td = curthread;
 657
 658         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 659                 xiovec->iov_base = beio->sg_segs[i].addr;
 660                 xiovec->iov_len = beio->sg_segs[i].len;
 661         }
 662
 663         binuptime(&beio->ds_t0);
 664         mtx_lock(&be_lun->io_lock);
 665         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 666         mtx_unlock(&be_lun->io_lock);
 667
 668         if (beio->bio_cmd == BIO_READ) {
 669                 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 670
 671                 /*
 672                  * UFS pays attention to IO_DIRECT for reads.  If the
 673                  * DIRECTIO option is configured into the kernel, it calls
 674                  * ffs_rawread().  But that only works for single-segment
 675                  * uios with user space addresses.  In our case, with a
 676                  * kernel uio, it still reads into the buffer cache, but it
 677                  * will just try to release the buffer from the cache later
 678                  * on in ffs_read().
 679                  *
 680                  * ZFS does not pay attention to IO_DIRECT for reads.
 681                  *
 682                  * UFS does not pay attention to IO_SYNC for reads.
 683                  *
 684                  * ZFS pays attention to IO_SYNC (which translates into the
 685                  * Solaris define FRSYNC for zfs_read()) for reads.  It
 686                  * attempts to sync the file before reading.
 687                  */
 688                 error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred);
 689
 690                 VOP_UNLOCK(be_lun->vn, 0);
 691                 SDT_PROBE0(cbb, , read, file_done);
 692                 if (error == 0 && xuio.uio_resid > 0) {
 693                         /*
 694                          * If we red less then requested (EOF), then
 695                          * we should clean the rest of the buffer.
 696                          */
 697                         s = beio->io_len - xuio.uio_resid;
 698                         for (i = 0; i < beio->num_segs; i++) {
 699                                 if (s >= beio->sg_segs[i].len) {
 700                                         s -= beio->sg_segs[i].len;
 701                                         continue;
 702                                 }
 703                                 bzero((uint8_t *)beio->sg_segs[i].addr + s,
 704                                     beio->sg_segs[i].len - s);
 705                                 s = 0;
 706                         }
 707                 }
 708         } else {
 709                 struct mount *mountpoint;
 710                 int lock_flags;
 711
 712                 (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 713
 714                 if (MNT_SHARED_WRITES(mountpoint) || ((mountpoint == NULL)
 715                   && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 716                         lock_flags = LK_SHARED;
 717                 else
 718                         lock_flags = LK_EXCLUSIVE;
 719                 vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 720
 721                 /*
 722                  * UFS pays attention to IO_DIRECT for writes.  The write
 723                  * is done asynchronously.  (Normally the write would just
 724                  * get put into cache.
 725                  *
 726                  * UFS pays attention to IO_SYNC for writes.  It will
 727                  * attempt to write the buffer out synchronously if that
 728                  * flag is set.
 729                  *
 730                  * ZFS does not pay attention to IO_DIRECT for writes.
 731                  *
 732                  * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC)
 733                  * for writes.  It will flush the transaction from the
 734                  * cache before returning.
 735                  */
 736                 error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred);
 737                 VOP_UNLOCK(be_lun->vn, 0);
 738
 739                 vn_finished_write(mountpoint);
 740                 SDT_PROBE0(cbb, , write, file_done);
 741         }
 742
 743         mtx_lock(&be_lun->io_lock);
 744         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 745             beio->ds_tag_type, beio->ds_trans_type,
 746             /*now*/ NULL, /*then*/&beio->ds_t0);
 747         mtx_unlock(&be_lun->io_lock);
 748
 749         /*
 750          * If we got an error, set the sense data to "MEDIUM ERROR" and
 751          * return the I/O to the user.
 752          */
 753         if (error != 0) {
 754                 if (error == ENOSPC || error == EDQUOT) {
 755                         ctl_set_space_alloc_fail(&io->scsiio);
 756                 } else if (error == EROFS || error == EACCES) {
 757                         ctl_set_hw_write_protected(&io->scsiio);
 758                 } else {
 759                         ctl_set_medium_error(&io->scsiio,
 760                             beio->bio_cmd == BIO_READ);
 761                 }
 762                 ctl_complete_beio(beio);
 763                 return;
 764         }
 765
 766         /*
 767          * If this is a write or a verify, we're all done.
 768          * If this is a read, we can now send the data to the user.
 769          */
 770         if ((beio->bio_cmd == BIO_WRITE) ||
 771             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 772                 ctl_set_success(&io->scsiio);
 773                 ctl_complete_beio(beio);
 774         } else {
 775                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 776                     beio->beio_cont == NULL) {
 777                         ctl_set_success(&io->scsiio);
 778                         ctl_serseq_done(io);
 779                 }
 780 #ifdef CTL_TIME_IO
 781                 getbinuptime(&io->io_hdr.dma_start_bt);
 782 #endif
 783                 ctl_datamove(io);
 784         }
 785 }
 786
 787 static void
 788 ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
 789                         struct ctl_be_block_io *beio)
 790 {
 791         union ctl_io *io = beio->io;
 792         struct ctl_lba_len_flags *lbalen = ARGS(io);
 793         struct scsi_get_lba_status_data *data;
 794         off_t roff, off;
 795         int error, status;
 796
 797         DPRINTF("entered\n");
 798
 799         off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
 800         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 801         error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off,
 802             0, curthread->td_ucred, curthread);
 803         if (error == 0 && off > roff)
 804                 status = 0;     /* mapped up to off */
 805         else {
 806                 error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off,
 807                     0, curthread->td_ucred, curthread);
 808                 if (error == 0 && off > roff)
 809                         status = 1;     /* deallocated up to off */
 810                 else {
 811                         status = 0;     /* unknown up to the end */
 812                         off = be_lun->size_bytes;
 813                 }
 814         }
 815         VOP_UNLOCK(be_lun->vn, 0);
 816
 817         data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
 818         scsi_u64to8b(lbalen->lba, data->descr[0].addr);
 819         scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
 820             lbalen->lba), data->descr[0].length);
 821         data->descr[0].status = status;
 822
 823         ctl_complete_beio(beio);
 824 }
 825
 826 static uint64_t
 827 ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname)
 828 {
 829         struct vattr            vattr;
 830         struct statfs           statfs;
 831         uint64_t                val;
 832         int                     error;
 833
 834         val = UINT64_MAX;
 835         if (be_lun->vn == NULL)
 836                 return (val);
 837         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 838         if (strcmp(attrname, "blocksused") == 0) {
 839                 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
 840                 if (error == 0)
 841                         val = vattr.va_bytes / be_lun->cbe_lun.blocksize;
 842         }
 843         if (strcmp(attrname, "blocksavail") == 0 &&
 844             (be_lun->vn->v_iflag & VI_DOOMED) == 0) {
 845                 error = VFS_STATFS(be_lun->vn->v_mount, &statfs);
 846                 if (error == 0)
 847                         val = statfs.f_bavail * statfs.f_bsize /
 848                             be_lun->cbe_lun.blocksize;
 849         }
 850         VOP_UNLOCK(be_lun->vn, 0);
 851         return (val);
 852 }
 853
 854 static void
 855 ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
 856                            struct ctl_be_block_io *beio)
 857 {
 858         union ctl_io *io;
 859         struct cdevsw *csw;
 860         struct cdev *dev;
 861         struct uio xuio;
 862         struct iovec *xiovec;
 863         int error, flags, i, ref;
 864
 865         DPRINTF("entered\n");
 866
 867         io = beio->io;
 868         flags = 0;
 869         if (ARGS(io)->flags & CTL_LLF_DPO)
 870                 flags |= IO_DIRECT;
 871         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 872                 flags |= IO_SYNC;
 873
 874         bzero(&xuio, sizeof(xuio));
 875         if (beio->bio_cmd == BIO_READ) {
 876                 SDT_PROBE0(cbb, , read, file_start);
 877                 xuio.uio_rw = UIO_READ;
 878         } else {
 879                 SDT_PROBE0(cbb, , write, file_start);
 880                 xuio.uio_rw = UIO_WRITE;
 881         }
 882         xuio.uio_offset = beio->io_offset;
 883         xuio.uio_resid = beio->io_len;
 884         xuio.uio_segflg = UIO_SYSSPACE;
 885         xuio.uio_iov = beio->xiovecs;
 886         xuio.uio_iovcnt = beio->num_segs;
 887         xuio.uio_td = curthread;
 888
 889         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 890                 xiovec->iov_base = beio->sg_segs[i].addr;
 891                 xiovec->iov_len = beio->sg_segs[i].len;
 892         }
 893
 894         binuptime(&beio->ds_t0);
 895         mtx_lock(&be_lun->io_lock);
 896         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 897         mtx_unlock(&be_lun->io_lock);
 898
 899         csw = devvn_refthread(be_lun->vn, &dev, &ref);
 900         if (csw) {
 901                 if (beio->bio_cmd == BIO_READ)
 902                         error = csw->d_read(dev, &xuio, flags);
 903                 else
 904                         error = csw->d_write(dev, &xuio, flags);
 905                 dev_relthread(dev, ref);
 906         } else
 907                 error = ENXIO;
 908
 909         if (beio->bio_cmd == BIO_READ)
 910                 SDT_PROBE0(cbb, , read, file_done);
 911         else
 912                 SDT_PROBE0(cbb, , write, file_done);
 913
 914         mtx_lock(&be_lun->io_lock);
 915         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 916             beio->ds_tag_type, beio->ds_trans_type,
 917             /*now*/ NULL, /*then*/&beio->ds_t0);
 918         mtx_unlock(&be_lun->io_lock);
 919
 920         /*
 921          * If we got an error, set the sense data to "MEDIUM ERROR" and
 922          * return the I/O to the user.
 923          */
 924         if (error != 0) {
 925                 if (error == ENOSPC || error == EDQUOT) {
 926                         ctl_set_space_alloc_fail(&io->scsiio);
 927                 } else if (error == EROFS || error == EACCES) {
 928                         ctl_set_hw_write_protected(&io->scsiio);
 929                 } else {
 930                         ctl_set_medium_error(&io->scsiio,
 931                             beio->bio_cmd == BIO_READ);
 932                 }
 933                 ctl_complete_beio(beio);
 934                 return;
 935         }
 936
 937         /*
 938          * If this is a write or a verify, we're all done.
 939          * If this is a read, we can now send the data to the user.
 940          */
 941         if ((beio->bio_cmd == BIO_WRITE) ||
 942             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 943                 ctl_set_success(&io->scsiio);
 944                 ctl_complete_beio(beio);
 945         } else {
 946                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 947                     beio->beio_cont == NULL) {
 948                         ctl_set_success(&io->scsiio);
 949                         ctl_serseq_done(io);
 950                 }
 951 #ifdef CTL_TIME_IO
 952                 getbinuptime(&io->io_hdr.dma_start_bt);
 953 #endif
 954                 ctl_datamove(io);
 955         }
 956 }
 957
 958 static void
 959 ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun,
 960                         struct ctl_be_block_io *beio)
 961 {
 962         union ctl_io *io = beio->io;
 963         struct cdevsw *csw;
 964         struct cdev *dev;
 965         struct ctl_lba_len_flags *lbalen = ARGS(io);
 966         struct scsi_get_lba_status_data *data;
 967         off_t roff, off;
 968         int error, ref, status;
 969
 970         DPRINTF("entered\n");
 971
 972         csw = devvn_refthread(be_lun->vn, &dev, &ref);
 973         if (csw == NULL) {
 974                 status = 0;     /* unknown up to the end */
 975                 off = be_lun->size_bytes;
 976                 goto done;
 977         }
 978         off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
 979         error = csw->d_ioctl(dev, FIOSEEKHOLE, (caddr_t)&off, FREAD,
 980             curthread);
 981         if (error == 0 && off > roff)
 982                 status = 0;     /* mapped up to off */
 983         else {
 984                 error = csw->d_ioctl(dev, FIOSEEKDATA, (caddr_t)&off, FREAD,
 985                     curthread);
 986                 if (error == 0 && off > roff)
 987                         status = 1;     /* deallocated up to off */
 988                 else {
 989                         status = 0;     /* unknown up to the end */
 990                         off = be_lun->size_bytes;
 991                 }
 992         }
 993         dev_relthread(dev, ref);
 994
 995 done:
 996         data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
 997         scsi_u64to8b(lbalen->lba, data->descr[0].addr);
 998         scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
 999             lbalen->lba), data->descr[0].length);
1000         data->descr[0].status = status;
1001
1002         ctl_complete_beio(beio);
1003 }
1004
1005 static void
1006 ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
1007                        struct ctl_be_block_io *beio)
1008 {
1009         struct bio *bio;
1010         struct cdevsw *csw;
1011         struct cdev *dev;
1012         int ref;
1013
1014         DPRINTF("entered\n");
1015
1016         /* This can't fail, it's a blocking allocation. */
1017         bio = g_alloc_bio();
1018
1019         bio->bio_cmd        = BIO_FLUSH;
1020         bio->bio_offset     = 0;
1021         bio->bio_data       = 0;
1022         bio->bio_done       = ctl_be_block_biodone;
1023         bio->bio_caller1    = beio;
1024         bio->bio_pblkno     = 0;
1025
1026         /*
1027          * We don't need to acquire the LUN lock here, because we are only
1028          * sending one bio, and so there is no other context to synchronize
1029          * with.
1030          */
1031         beio->num_bios_sent = 1;
1032         beio->send_complete = 1;
1033
1034         binuptime(&beio->ds_t0);
1035         mtx_lock(&be_lun->io_lock);
1036         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1037         mtx_unlock(&be_lun->io_lock);
1038
1039         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1040         if (csw) {
1041                 bio->bio_dev = dev;
1042                 csw->d_strategy(bio);
1043                 dev_relthread(dev, ref);
1044         } else {
1045                 bio->bio_error = ENXIO;
1046                 ctl_be_block_biodone(bio);
1047         }
1048 }
1049
1050 static void
1051 ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun,
1052                        struct ctl_be_block_io *beio,
1053                        uint64_t off, uint64_t len, int last)
1054 {
1055         struct bio *bio;
1056         uint64_t maxlen;
1057         struct cdevsw *csw;
1058         struct cdev *dev;
1059         int ref;
1060
1061         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1062         maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize);
1063         while (len > 0) {
1064                 bio = g_alloc_bio();
1065                 bio->bio_cmd        = BIO_DELETE;
1066                 bio->bio_dev        = dev;
1067                 bio->bio_offset     = off;
1068                 bio->bio_length     = MIN(len, maxlen);
1069                 bio->bio_data       = 0;
1070                 bio->bio_done       = ctl_be_block_biodone;
1071                 bio->bio_caller1    = beio;
1072                 bio->bio_pblkno     = off / be_lun->cbe_lun.blocksize;
1073
1074                 off += bio->bio_length;
1075                 len -= bio->bio_length;
1076
1077                 mtx_lock(&be_lun->io_lock);
1078                 beio->num_bios_sent++;
1079                 if (last && len == 0)
1080                         beio->send_complete = 1;
1081                 mtx_unlock(&be_lun->io_lock);
1082
1083                 if (csw) {
1084                         csw->d_strategy(bio);
1085                 } else {
1086                         bio->bio_error = ENXIO;
1087                         ctl_be_block_biodone(bio);
1088                 }
1089         }
1090         if (csw)
1091                 dev_relthread(dev, ref);
1092 }
1093
1094 static void
1095 ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
1096                        struct ctl_be_block_io *beio)
1097 {
1098         union ctl_io *io;
1099         struct ctl_ptr_len_flags *ptrlen;
1100         struct scsi_unmap_desc *buf, *end;
1101         uint64_t len;
1102
1103         io = beio->io;
1104
1105         DPRINTF("entered\n");
1106
1107         binuptime(&beio->ds_t0);
1108         mtx_lock(&be_lun->io_lock);
1109         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1110         mtx_unlock(&be_lun->io_lock);
1111
1112         if (beio->io_offset == -1) {
1113                 beio->io_len = 0;
1114                 ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1115                 buf = (struct scsi_unmap_desc *)ptrlen->ptr;
1116                 end = buf + ptrlen->len / sizeof(*buf);
1117                 for (; buf < end; buf++) {
1118                         len = (uint64_t)scsi_4btoul(buf->length) *
1119                             be_lun->cbe_lun.blocksize;
1120                         beio->io_len += len;
1121                         ctl_be_block_unmap_dev_range(be_lun, beio,
1122                             scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize,
1123                             len, (end - buf < 2) ? TRUE : FALSE);
1124                 }
1125         } else
1126                 ctl_be_block_unmap_dev_range(be_lun, beio,
1127                     beio->io_offset, beio->io_len, TRUE);
1128 }
1129
1130 static void
1131 ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
1132                           struct ctl_be_block_io *beio)
1133 {
1134         TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue);
1135         struct bio *bio;
1136         struct cdevsw *csw;
1137         struct cdev *dev;
1138         off_t cur_offset;
1139         int i, max_iosize, ref;
1140
1141         DPRINTF("entered\n");
1142         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1143
1144         /*
1145          * We have to limit our I/O size to the maximum supported by the
1146          * backend device.  Hopefully it is MAXPHYS.  If the driver doesn't
1147          * set it properly, use DFLTPHYS.
1148          */
1149         if (csw) {
1150                 max_iosize = dev->si_iosize_max;
1151                 if (max_iosize < PAGE_SIZE)
1152                         max_iosize = DFLTPHYS;
1153         } else
1154                 max_iosize = DFLTPHYS;
1155
1156         cur_offset = beio->io_offset;
1157         for (i = 0; i < beio->num_segs; i++) {
1158                 size_t cur_size;
1159                 uint8_t *cur_ptr;
1160
1161                 cur_size = beio->sg_segs[i].len;
1162                 cur_ptr = beio->sg_segs[i].addr;
1163
1164                 while (cur_size > 0) {
1165                         /* This can't fail, it's a blocking allocation. */
1166                         bio = g_alloc_bio();
1167
1168                         KASSERT(bio != NULL, ("g_alloc_bio() failed!\n"));
1169
1170                         bio->bio_cmd = beio->bio_cmd;
1171                         bio->bio_dev = dev;
1172                         bio->bio_caller1 = beio;
1173                         bio->bio_length = min(cur_size, max_iosize);
1174                         bio->bio_offset = cur_offset;
1175                         bio->bio_data = cur_ptr;
1176                         bio->bio_done = ctl_be_block_biodone;
1177                         bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize;
1178
1179                         cur_offset += bio->bio_length;
1180                         cur_ptr += bio->bio_length;
1181                         cur_size -= bio->bio_length;
1182
1183                         TAILQ_INSERT_TAIL(&queue, bio, bio_queue);
1184                         beio->num_bios_sent++;
1185                 }
1186         }
1187         binuptime(&beio->ds_t0);
1188         mtx_lock(&be_lun->io_lock);
1189         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1190         beio->send_complete = 1;
1191         mtx_unlock(&be_lun->io_lock);
1192
1193         /*
1194          * Fire off all allocated requests!
1195          */
1196         while ((bio = TAILQ_FIRST(&queue)) != NULL) {
1197                 TAILQ_REMOVE(&queue, bio, bio_queue);
1198                 if (csw)
1199                         csw->d_strategy(bio);
1200                 else {
1201                         bio->bio_error = ENXIO;
1202                         ctl_be_block_biodone(bio);
1203                 }
1204         }
1205         if (csw)
1206                 dev_relthread(dev, ref);
1207 }
1208
1209 static uint64_t
1210 ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname)
1211 {
1212         struct diocgattr_arg    arg;
1213         struct cdevsw *csw;
1214         struct cdev *dev;
1215         int error, ref;
1216
1217         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1218         if (csw == NULL)
1219                 return (UINT64_MAX);
1220         strlcpy(arg.name, attrname, sizeof(arg.name));
1221         arg.len = sizeof(arg.value.off);
1222         if (csw->d_ioctl) {
1223                 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
1224                     curthread);
1225         } else
1226                 error = ENODEV;
1227         dev_relthread(dev, ref);
1228         if (error != 0)
1229                 return (UINT64_MAX);
1230         return (arg.value.off);
1231 }
1232
1233 static void
1234 ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun,
1235                             union ctl_io *io)
1236 {
1237         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1238         struct ctl_be_block_io *beio;
1239         struct ctl_lba_len_flags *lbalen;
1240
1241         DPRINTF("entered\n");
1242         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1243         lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1244
1245         beio->io_len = lbalen->len * cbe_lun->blocksize;
1246         beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1247         beio->io_arg = (lbalen->flags & SSC_IMMED) != 0;
1248         beio->bio_cmd = BIO_FLUSH;
1249         beio->ds_trans_type = DEVSTAT_NO_DATA;
1250         DPRINTF("SYNC\n");
1251         be_lun->lun_flush(be_lun, beio);
1252 }
1253
1254 static void
1255 ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio)
1256 {
1257         union ctl_io *io;
1258
1259         io = beio->io;
1260         ctl_free_beio(beio);
1261         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1262             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1263              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1264                 ctl_config_write_done(io);
1265                 return;
1266         }
1267
1268         ctl_be_block_config_write(io);
1269 }
1270
1271 static void
1272 ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,
1273                             union ctl_io *io)
1274 {
1275         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1276         struct ctl_be_block_io *beio;
1277         struct ctl_lba_len_flags *lbalen;
1278         uint64_t len_left, lba;
1279         uint32_t pb, pbo, adj;
1280         int i, seglen;
1281         uint8_t *buf, *end;
1282
1283         DPRINTF("entered\n");
1284
1285         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1286         lbalen = ARGS(beio->io);
1287
1288         if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) ||
1289             (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) {
1290                 ctl_free_beio(beio);
1291                 ctl_set_invalid_field(&io->scsiio,
1292                                       /*sks_valid*/ 1,
1293                                       /*command*/ 1,
1294                                       /*field*/ 1,
1295                                       /*bit_valid*/ 0,
1296                                       /*bit*/ 0);
1297                 ctl_config_write_done(io);
1298                 return;
1299         }
1300
1301         if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) {
1302                 beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1303                 beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize;
1304                 beio->bio_cmd = BIO_DELETE;
1305                 beio->ds_trans_type = DEVSTAT_FREE;
1306
1307                 be_lun->unmap(be_lun, beio);
1308                 return;
1309         }
1310
1311         beio->bio_cmd = BIO_WRITE;
1312         beio->ds_trans_type = DEVSTAT_WRITE;
1313
1314         DPRINTF("WRITE SAME at LBA %jx len %u\n",
1315                (uintmax_t)lbalen->lba, lbalen->len);
1316
1317         pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp;
1318         if (be_lun->cbe_lun.pblockoff > 0)
1319                 pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff;
1320         else
1321                 pbo = 0;
1322         len_left = (uint64_t)lbalen->len * cbe_lun->blocksize;
1323         for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) {
1324
1325                 /*
1326                  * Setup the S/G entry for this chunk.
1327                  */
1328                 seglen = MIN(CTLBLK_MAX_SEG, len_left);
1329                 if (pb > cbe_lun->blocksize) {
1330                         adj = ((lbalen->lba + lba) * cbe_lun->blocksize +
1331                             seglen - pbo) % pb;
1332                         if (seglen > adj)
1333                                 seglen -= adj;
1334                         else
1335                                 seglen -= seglen % cbe_lun->blocksize;
1336                 } else
1337                         seglen -= seglen % cbe_lun->blocksize;
1338                 beio->sg_segs[i].len = seglen;
1339                 beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1340
1341                 DPRINTF("segment %d addr %p len %zd\n", i,
1342                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1343
1344                 beio->num_segs++;
1345                 len_left -= seglen;
1346
1347                 buf = beio->sg_segs[i].addr;
1348                 end = buf + seglen;
1349                 for (; buf < end; buf += cbe_lun->blocksize) {
1350                         if (lbalen->flags & SWS_NDOB) {
1351                                 memset(buf, 0, cbe_lun->blocksize);
1352                         } else {
1353                                 memcpy(buf, io->scsiio.kern_data_ptr,
1354                                     cbe_lun->blocksize);
1355                         }
1356                         if (lbalen->flags & SWS_LBDATA)
1357                                 scsi_ulto4b(lbalen->lba + lba, buf);
1358                         lba++;
1359                 }
1360         }
1361
1362         beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1363         beio->io_len = lba * cbe_lun->blocksize;
1364
1365         /* We can not do all in one run. Correct and schedule rerun. */
1366         if (len_left > 0) {
1367                 lbalen->lba += lba;
1368                 lbalen->len -= lba;
1369                 beio->beio_cont = ctl_be_block_cw_done_ws;
1370         }
1371
1372         be_lun->dispatch(be_lun, beio);
1373 }
1374
1375 static void
1376 ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun,
1377                             union ctl_io *io)
1378 {
1379         struct ctl_be_block_io *beio;
1380         struct ctl_ptr_len_flags *ptrlen;
1381
1382         DPRINTF("entered\n");
1383
1384         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1385         ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1386
1387         if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) {
1388                 ctl_free_beio(beio);
1389                 ctl_set_invalid_field(&io->scsiio,
1390                                       /*sks_valid*/ 0,
1391                                       /*command*/ 1,
1392                                       /*field*/ 0,
1393                                       /*bit_valid*/ 0,
1394                                       /*bit*/ 0);
1395                 ctl_config_write_done(io);
1396                 return;
1397         }
1398
1399         beio->io_len = 0;
1400         beio->io_offset = -1;
1401         beio->bio_cmd = BIO_DELETE;
1402         beio->ds_trans_type = DEVSTAT_FREE;
1403         DPRINTF("UNMAP\n");
1404         be_lun->unmap(be_lun, beio);
1405 }
1406
1407 static void
1408 ctl_be_block_cr_done(struct ctl_be_block_io *beio)
1409 {
1410         union ctl_io *io;
1411
1412         io = beio->io;
1413         ctl_free_beio(beio);
1414         ctl_config_read_done(io);
1415 }
1416
1417 static void
1418 ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
1419                          union ctl_io *io)
1420 {
1421         struct ctl_be_block_io *beio;
1422         struct ctl_be_block_softc *softc;
1423
1424         DPRINTF("entered\n");
1425
1426         softc = be_lun->softc;
1427         beio = ctl_alloc_beio(softc);
1428         beio->io = io;
1429         beio->lun = be_lun;
1430         beio->beio_cont = ctl_be_block_cr_done;
1431         PRIV(io)->ptr = (void *)beio;
1432
1433         switch (io->scsiio.cdb[0]) {
1434         case SERVICE_ACTION_IN:         /* GET LBA STATUS */
1435                 beio->bio_cmd = -1;
1436                 beio->ds_trans_type = DEVSTAT_NO_DATA;
1437                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1438                 beio->io_len = 0;
1439                 if (be_lun->get_lba_status)
1440                         be_lun->get_lba_status(be_lun, beio);
1441                 else
1442                         ctl_be_block_cr_done(beio);
1443                 break;
1444         default:
1445                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1446                 break;
1447         }
1448 }
1449
1450 static void
1451 ctl_be_block_cw_done(struct ctl_be_block_io *beio)
1452 {
1453         union ctl_io *io;
1454
1455         io = beio->io;
1456         ctl_free_beio(beio);
1457         ctl_config_write_done(io);
1458 }
1459
1460 static void
1461 ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
1462                          union ctl_io *io)
1463 {
1464         struct ctl_be_block_io *beio;
1465         struct ctl_be_block_softc *softc;
1466
1467         DPRINTF("entered\n");
1468
1469         softc = be_lun->softc;
1470         beio = ctl_alloc_beio(softc);
1471         beio->io = io;
1472         beio->lun = be_lun;
1473         beio->beio_cont = ctl_be_block_cw_done;
1474         switch (io->scsiio.tag_type) {
1475         case CTL_TAG_ORDERED:
1476                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1477                 break;
1478         case CTL_TAG_HEAD_OF_QUEUE:
1479                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1480                 break;
1481         case CTL_TAG_UNTAGGED:
1482         case CTL_TAG_SIMPLE:
1483         case CTL_TAG_ACA:
1484         default:
1485                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1486                 break;
1487         }
1488         PRIV(io)->ptr = (void *)beio;
1489
1490         switch (io->scsiio.cdb[0]) {
1491         case SYNCHRONIZE_CACHE:
1492         case SYNCHRONIZE_CACHE_16:
1493                 ctl_be_block_cw_dispatch_sync(be_lun, io);
1494                 break;
1495         case WRITE_SAME_10:
1496         case WRITE_SAME_16:
1497                 ctl_be_block_cw_dispatch_ws(be_lun, io);
1498                 break;
1499         case UNMAP:
1500                 ctl_be_block_cw_dispatch_unmap(be_lun, io);
1501                 break;
1502         default:
1503                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1504                 break;
1505         }
1506 }
1507
1508 SDT_PROBE_DEFINE1(cbb, , read, start, "uint64_t");
1509 SDT_PROBE_DEFINE1(cbb, , write, start, "uint64_t");
1510 SDT_PROBE_DEFINE1(cbb, , read, alloc_done, "uint64_t");
1511 SDT_PROBE_DEFINE1(cbb, , write, alloc_done, "uint64_t");
1512
1513 static void
1514 ctl_be_block_next(struct ctl_be_block_io *beio)
1515 {
1516         struct ctl_be_block_lun *be_lun;
1517         union ctl_io *io;
1518
1519         io = beio->io;
1520         be_lun = beio->lun;
1521         ctl_free_beio(beio);
1522         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1523             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1524              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1525                 ctl_data_submit_done(io);
1526                 return;
1527         }
1528
1529         io->io_hdr.status &= ~CTL_STATUS_MASK;
1530         io->io_hdr.status |= CTL_STATUS_NONE;
1531
1532         mtx_lock(&be_lun->queue_lock);
1533         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1534         mtx_unlock(&be_lun->queue_lock);
1535         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1536 }
1537
1538 static void
1539 ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
1540                            union ctl_io *io)
1541 {
1542         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1543         struct ctl_be_block_io *beio;
1544         struct ctl_be_block_softc *softc;
1545         struct ctl_lba_len_flags *lbalen;
1546         struct ctl_ptr_len_flags *bptrlen;
1547         uint64_t len_left, lbas;
1548         int i;
1549
1550         softc = be_lun->softc;
1551
1552         DPRINTF("entered\n");
1553
1554         lbalen = ARGS(io);
1555         if (lbalen->flags & CTL_LLF_WRITE) {
1556                 SDT_PROBE0(cbb, , write, start);
1557         } else {
1558                 SDT_PROBE0(cbb, , read, start);
1559         }
1560
1561         beio = ctl_alloc_beio(softc);
1562         beio->io = io;
1563         beio->lun = be_lun;
1564         bptrlen = PRIV(io);
1565         bptrlen->ptr = (void *)beio;
1566
1567         switch (io->scsiio.tag_type) {
1568         case CTL_TAG_ORDERED:
1569                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1570                 break;
1571         case CTL_TAG_HEAD_OF_QUEUE:
1572                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1573                 break;
1574         case CTL_TAG_UNTAGGED:
1575         case CTL_TAG_SIMPLE:
1576         case CTL_TAG_ACA:
1577         default:
1578                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1579                 break;
1580         }
1581
1582         if (lbalen->flags & CTL_LLF_WRITE) {
1583                 beio->bio_cmd = BIO_WRITE;
1584                 beio->ds_trans_type = DEVSTAT_WRITE;
1585         } else {
1586                 beio->bio_cmd = BIO_READ;
1587                 beio->ds_trans_type = DEVSTAT_READ;
1588         }
1589
1590         DPRINTF("%s at LBA %jx len %u @%ju\n",
1591                (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
1592                (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len);
1593         if (lbalen->flags & CTL_LLF_COMPARE)
1594                 lbas = CTLBLK_HALF_IO_SIZE;
1595         else
1596                 lbas = CTLBLK_MAX_IO_SIZE;
1597         lbas = MIN(lbalen->len - bptrlen->len, lbas / cbe_lun->blocksize);
1598         beio->io_offset = (lbalen->lba + bptrlen->len) * cbe_lun->blocksize;
1599         beio->io_len = lbas * cbe_lun->blocksize;
1600         bptrlen->len += lbas;
1601
1602         for (i = 0, len_left = beio->io_len; len_left > 0; i++) {
1603                 KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)",
1604                     i, CTLBLK_MAX_SEGS));
1605
1606                 /*
1607                  * Setup the S/G entry for this chunk.
1608                  */
1609                 beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left);
1610                 beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1611
1612                 DPRINTF("segment %d addr %p len %zd\n", i,
1613                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1614
1615                 /* Set up second segment for compare operation. */
1616                 if (lbalen->flags & CTL_LLF_COMPARE) {
1617                         beio->sg_segs[i + CTLBLK_HALF_SEGS].len =
1618                             beio->sg_segs[i].len;
1619                         beio->sg_segs[i + CTLBLK_HALF_SEGS].addr =
1620                             uma_zalloc(be_lun->lun_zone, M_WAITOK);
1621                 }
1622
1623                 beio->num_segs++;
1624                 len_left -= beio->sg_segs[i].len;
1625         }
1626         if (bptrlen->len < lbalen->len)
1627                 beio->beio_cont = ctl_be_block_next;
1628         io->scsiio.be_move_done = ctl_be_block_move_done;
1629         /* For compare we have separate S/G lists for read and datamove. */
1630         if (lbalen->flags & CTL_LLF_COMPARE)
1631                 io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS];
1632         else
1633                 io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
1634         io->scsiio.kern_data_len = beio->io_len;
1635         io->scsiio.kern_sg_entries = beio->num_segs;
1636         io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
1637
1638         /*
1639          * For the read case, we need to read the data into our buffers and
1640          * then we can send it back to the user.  For the write case, we
1641          * need to get the data from the user first.
1642          */
1643         if (beio->bio_cmd == BIO_READ) {
1644                 SDT_PROBE0(cbb, , read, alloc_done);
1645                 be_lun->dispatch(be_lun, beio);
1646         } else {
1647                 SDT_PROBE0(cbb, , write, alloc_done);
1648 #ifdef CTL_TIME_IO
1649                 getbinuptime(&io->io_hdr.dma_start_bt);
1650 #endif
1651                 ctl_datamove(io);
1652         }
1653 }
1654
1655 static void
1656 ctl_be_block_worker(void *context, int pending)
1657 {
1658         struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context;
1659         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1660         union ctl_io *io;
1661         struct ctl_be_block_io *beio;
1662
1663         DPRINTF("entered\n");
1664         /*
1665          * Fetch and process I/Os from all queues.  If we detect LUN
1666          * CTL_LUN_FLAG_NO_MEDIA status here -- it is result of a race,
1667          * so make response maximally opaque to not confuse initiator.
1668          */
1669         for (;;) {
1670                 mtx_lock(&be_lun->queue_lock);
1671                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);
1672                 if (io != NULL) {
1673                         DPRINTF("datamove queue\n");
1674                         STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr,
1675                                       ctl_io_hdr, links);
1676                         mtx_unlock(&be_lun->queue_lock);
1677                         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1678                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1679                                 ctl_set_busy(&io->scsiio);
1680                                 ctl_complete_beio(beio);
1681                                 return;
1682                         }
1683                         be_lun->dispatch(be_lun, beio);
1684                         continue;
1685                 }
1686                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);
1687                 if (io != NULL) {
1688                         DPRINTF("config write queue\n");
1689                         STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr,
1690                                       ctl_io_hdr, links);
1691                         mtx_unlock(&be_lun->queue_lock);
1692                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1693                                 ctl_set_busy(&io->scsiio);
1694                                 ctl_config_write_done(io);
1695                                 return;
1696                         }
1697                         ctl_be_block_cw_dispatch(be_lun, io);
1698                         continue;
1699                 }
1700                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue);
1701                 if (io != NULL) {
1702                         DPRINTF("config read queue\n");
1703                         STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr,
1704                                       ctl_io_hdr, links);
1705                         mtx_unlock(&be_lun->queue_lock);
1706                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1707                                 ctl_set_busy(&io->scsiio);
1708                                 ctl_config_read_done(io);
1709                                 return;
1710                         }
1711                         ctl_be_block_cr_dispatch(be_lun, io);
1712                         continue;
1713                 }
1714                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);
1715                 if (io != NULL) {
1716                         DPRINTF("input queue\n");
1717                         STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr,
1718                                       ctl_io_hdr, links);
1719                         mtx_unlock(&be_lun->queue_lock);
1720                         if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
1721                                 ctl_set_busy(&io->scsiio);
1722                                 ctl_data_submit_done(io);
1723                                 return;
1724                         }
1725                         ctl_be_block_dispatch(be_lun, io);
1726                         continue;
1727                 }
1728
1729                 /*
1730                  * If we get here, there is no work left in the queues, so
1731                  * just break out and let the task queue go to sleep.
1732                  */
1733                 mtx_unlock(&be_lun->queue_lock);
1734                 break;
1735         }
1736 }
1737
1738 /*
1739  * Entry point from CTL to the backend for I/O.  We queue everything to a
1740  * work thread, so this just puts the I/O on a queue and wakes up the
1741  * thread.
1742  */
1743 static int
1744 ctl_be_block_submit(union ctl_io *io)
1745 {
1746         struct ctl_be_block_lun *be_lun;
1747         struct ctl_be_lun *cbe_lun;
1748
1749         DPRINTF("entered\n");
1750
1751         cbe_lun = CTL_BACKEND_LUN(io);
1752         be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
1753
1754         /*
1755          * Make sure we only get SCSI I/O.
1756          */
1757         KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type "
1758                 "%#x) encountered", io->io_hdr.io_type));
1759
1760         PRIV(io)->len = 0;
1761
1762         mtx_lock(&be_lun->queue_lock);
1763         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1764         mtx_unlock(&be_lun->queue_lock);
1765         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1766
1767         return (CTL_RETVAL_COMPLETE);
1768 }
1769
1770 static int
1771 ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
1772                         int flag, struct thread *td)
1773 {
1774         struct ctl_be_block_softc *softc;
1775         int error;
1776
1777         softc = &backend_block_softc;
1778
1779         error = 0;
1780
1781         switch (cmd) {
1782         case CTL_LUN_REQ: {
1783                 struct ctl_lun_req *lun_req;
1784
1785                 lun_req = (struct ctl_lun_req *)addr;
1786
1787                 switch (lun_req->reqtype) {
1788                 case CTL_LUNREQ_CREATE:
1789                         error = ctl_be_block_create(softc, lun_req);
1790                         break;
1791                 case CTL_LUNREQ_RM:
1792                         error = ctl_be_block_rm(softc, lun_req);
1793                         break;
1794                 case CTL_LUNREQ_MODIFY:
1795                         error = ctl_be_block_modify(softc, lun_req);
1796                         break;
1797                 default:
1798                         lun_req->status = CTL_LUN_ERROR;
1799                         snprintf(lun_req->error_str, sizeof(lun_req->error_str),
1800                                  "invalid LUN request type %d",
1801                                  lun_req->reqtype);
1802                         break;
1803                 }
1804                 break;
1805         }
1806         default:
1807                 error = ENOTTY;
1808                 break;
1809         }
1810
1811         return (error);
1812 }
1813
1814 static int
1815 ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1816 {
1817         struct ctl_be_lun *cbe_lun;
1818         struct ctl_be_block_filedata *file_data;
1819         struct ctl_lun_create_params *params;
1820         char                         *value;
1821         struct vattr                  vattr;
1822         off_t                         ps, pss, po, pos, us, uss, uo, uos;
1823         int                           error;
1824
1825         cbe_lun = &be_lun->cbe_lun;
1826         file_data = &be_lun->backend.file;
1827         params = &be_lun->params;
1828
1829         be_lun->dev_type = CTL_BE_BLOCK_FILE;
1830         be_lun->dispatch = ctl_be_block_dispatch_file;
1831         be_lun->lun_flush = ctl_be_block_flush_file;
1832         be_lun->get_lba_status = ctl_be_block_gls_file;
1833         be_lun->getattr = ctl_be_block_getattr_file;
1834         be_lun->unmap = NULL;
1835         cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
1836
1837         error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
1838         if (error != 0) {
1839                 snprintf(req->error_str, sizeof(req->error_str),
1840                          "error calling VOP_GETATTR() for file %s",
1841                          be_lun->dev_path);
1842                 return (error);
1843         }
1844
1845         file_data->cred = crhold(curthread->td_ucred);
1846         if (params->lun_size_bytes != 0)
1847                 be_lun->size_bytes = params->lun_size_bytes;
1848         else
1849                 be_lun->size_bytes = vattr.va_size;
1850
1851         /*
1852          * For files we can use any logical block size.  Prefer 512 bytes
1853          * for compatibility reasons.  If file's vattr.va_blocksize
1854          * (preferred I/O block size) is bigger and multiple to chosen
1855          * logical block size -- report it as physical block size.
1856          */
1857         if (params->blocksize_bytes != 0)
1858                 cbe_lun->blocksize = params->blocksize_bytes;
1859         else if (cbe_lun->lun_type == T_CDROM)
1860                 cbe_lun->blocksize = 2048;
1861         else
1862                 cbe_lun->blocksize = 512;
1863         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
1864         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
1865             0 : (be_lun->size_blocks - 1);
1866
1867         us = ps = vattr.va_blocksize;
1868         uo = po = 0;
1869
1870         value = ctl_get_opt(&cbe_lun->options, "pblocksize");
1871         if (value != NULL)
1872                 ctl_expand_number(value, &ps);
1873         value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
1874         if (value != NULL)
1875                 ctl_expand_number(value, &po);
1876         pss = ps / cbe_lun->blocksize;
1877         pos = po / cbe_lun->blocksize;
1878         if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
1879             ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
1880                 cbe_lun->pblockexp = fls(pss) - 1;
1881                 cbe_lun->pblockoff = (pss - pos) % pss;
1882         }
1883
1884         value = ctl_get_opt(&cbe_lun->options, "ublocksize");
1885         if (value != NULL)
1886                 ctl_expand_number(value, &us);
1887         value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
1888         if (value != NULL)
1889                 ctl_expand_number(value, &uo);
1890         uss = us / cbe_lun->blocksize;
1891         uos = uo / cbe_lun->blocksize;
1892         if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
1893             ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
1894                 cbe_lun->ublockexp = fls(uss) - 1;
1895                 cbe_lun->ublockoff = (uss - uos) % uss;
1896         }
1897
1898         /*
1899          * Sanity check.  The media size has to be at least one
1900          * sector long.
1901          */
1902         if (be_lun->size_bytes < cbe_lun->blocksize) {
1903                 error = EINVAL;
1904                 snprintf(req->error_str, sizeof(req->error_str),
1905                          "file %s size %ju < block size %u", be_lun->dev_path,
1906                          (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize);
1907         }
1908
1909         cbe_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / cbe_lun->blocksize;
1910         return (error);
1911 }
1912
1913 static int
1914 ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1915 {
1916         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1917         struct ctl_lun_create_params *params;
1918         struct cdevsw                *csw;
1919         struct cdev                  *dev;
1920         char                         *value;
1921         int                           error, atomic, maxio, ref, unmap, tmp;
1922         off_t                         ps, pss, po, pos, us, uss, uo, uos, otmp;
1923
1924         params = &be_lun->params;
1925
1926         be_lun->dev_type = CTL_BE_BLOCK_DEV;
1927         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1928         if (csw == NULL)
1929                 return (ENXIO);
1930         if (strcmp(csw->d_name, "zvol") == 0) {
1931                 be_lun->dispatch = ctl_be_block_dispatch_zvol;
1932                 be_lun->get_lba_status = ctl_be_block_gls_zvol;
1933                 atomic = maxio = CTLBLK_MAX_IO_SIZE;
1934         } else {
1935                 be_lun->dispatch = ctl_be_block_dispatch_dev;
1936                 be_lun->get_lba_status = NULL;
1937                 atomic = 0;
1938                 maxio = dev->si_iosize_max;
1939                 if (maxio <= 0)
1940                         maxio = DFLTPHYS;
1941                 if (maxio > CTLBLK_MAX_IO_SIZE)
1942                         maxio = CTLBLK_MAX_IO_SIZE;
1943         }
1944         be_lun->lun_flush = ctl_be_block_flush_dev;
1945         be_lun->getattr = ctl_be_block_getattr_dev;
1946         be_lun->unmap = ctl_be_block_unmap_dev;
1947
1948         if (!csw->d_ioctl) {
1949                 dev_relthread(dev, ref);
1950                 snprintf(req->error_str, sizeof(req->error_str),
1951                          "no d_ioctl for device %s!", be_lun->dev_path);
1952                 return (ENODEV);
1953         }
1954
1955         error = csw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD,
1956                                curthread);
1957         if (error) {
1958                 dev_relthread(dev, ref);
1959                 snprintf(req->error_str, sizeof(req->error_str),
1960                          "error %d returned for DIOCGSECTORSIZE ioctl "
1961                          "on %s!", error, be_lun->dev_path);
1962                 return (error);
1963         }
1964
1965         /*
1966          * If the user has asked for a blocksize that is greater than the
1967          * backing device's blocksize, we can do it only if the blocksize
1968          * the user is asking for is an even multiple of the underlying
1969          * device's blocksize.
1970          */
1971         if ((params->blocksize_bytes != 0) &&
1972             (params->blocksize_bytes >= tmp)) {
1973                 if (params->blocksize_bytes % tmp == 0) {
1974                         cbe_lun->blocksize = params->blocksize_bytes;
1975                 } else {
1976                         dev_relthread(dev, ref);
1977                         snprintf(req->error_str, sizeof(req->error_str),
1978                                  "requested blocksize %u is not an even "
1979                                  "multiple of backing device blocksize %u",
1980                                  params->blocksize_bytes, tmp);
1981                         return (EINVAL);
1982                 }
1983         } else if (params->blocksize_bytes != 0) {
1984                 dev_relthread(dev, ref);
1985                 snprintf(req->error_str, sizeof(req->error_str),
1986                          "requested blocksize %u < backing device "
1987                          "blocksize %u", params->blocksize_bytes, tmp);
1988                 return (EINVAL);
1989         } else if (cbe_lun->lun_type == T_CDROM)
1990                 cbe_lun->blocksize = MAX(tmp, 2048);
1991         else
1992                 cbe_lun->blocksize = tmp;
1993
1994         error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD,
1995                              curthread);
1996         if (error) {
1997                 dev_relthread(dev, ref);
1998                 snprintf(req->error_str, sizeof(req->error_str),
1999                          "error %d returned for DIOCGMEDIASIZE "
2000                          " ioctl on %s!", error,
2001                          be_lun->dev_path);
2002                 return (error);
2003         }
2004
2005         if (params->lun_size_bytes != 0) {
2006                 if (params->lun_size_bytes > otmp) {
2007                         dev_relthread(dev, ref);
2008                         snprintf(req->error_str, sizeof(req->error_str),
2009                                  "requested LUN size %ju > backing device "
2010                                  "size %ju",
2011                                  (uintmax_t)params->lun_size_bytes,
2012                                  (uintmax_t)otmp);
2013                         return (EINVAL);
2014                 }
2015
2016                 be_lun->size_bytes = params->lun_size_bytes;
2017         } else
2018                 be_lun->size_bytes = otmp;
2019         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
2020         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
2021             0 : (be_lun->size_blocks - 1);
2022
2023         error = csw->d_ioctl(dev, DIOCGSTRIPESIZE, (caddr_t)&ps, FREAD,
2024             curthread);
2025         if (error)
2026                 ps = po = 0;
2027         else {
2028                 error = csw->d_ioctl(dev, DIOCGSTRIPEOFFSET, (caddr_t)&po,
2029                     FREAD, curthread);
2030                 if (error)
2031                         po = 0;
2032         }
2033         us = ps;
2034         uo = po;
2035
2036         value = ctl_get_opt(&cbe_lun->options, "pblocksize");
2037         if (value != NULL)
2038                 ctl_expand_number(value, &ps);
2039         value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
2040         if (value != NULL)
2041                 ctl_expand_number(value, &po);
2042         pss = ps / cbe_lun->blocksize;
2043         pos = po / cbe_lun->blocksize;
2044         if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
2045             ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
2046                 cbe_lun->pblockexp = fls(pss) - 1;
2047                 cbe_lun->pblockoff = (pss - pos) % pss;
2048         }
2049
2050         value = ctl_get_opt(&cbe_lun->options, "ublocksize");
2051         if (value != NULL)
2052                 ctl_expand_number(value, &us);
2053         value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
2054         if (value != NULL)
2055                 ctl_expand_number(value, &uo);
2056         uss = us / cbe_lun->blocksize;
2057         uos = uo / cbe_lun->blocksize;
2058         if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
2059             ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
2060                 cbe_lun->ublockexp = fls(uss) - 1;
2061                 cbe_lun->ublockoff = (uss - uos) % uss;
2062         }
2063
2064         cbe_lun->atomicblock = atomic / cbe_lun->blocksize;
2065         cbe_lun->opttxferlen = maxio / cbe_lun->blocksize;
2066
2067         if (be_lun->dispatch == ctl_be_block_dispatch_zvol) {
2068                 unmap = 1;
2069         } else {
2070                 struct diocgattr_arg    arg;
2071
2072                 strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name));
2073                 arg.len = sizeof(arg.value.i);
2074                 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
2075                     curthread);
2076                 unmap = (error == 0) ? arg.value.i : 0;
2077         }
2078         value = ctl_get_opt(&cbe_lun->options, "unmap");
2079         if (value != NULL)
2080                 unmap = (strcmp(value, "on") == 0);
2081         if (unmap)
2082                 cbe_lun->flags |= CTL_LUN_FLAG_UNMAP;
2083         else
2084                 cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
2085
2086         dev_relthread(dev, ref);
2087         return (0);
2088 }
2089
2090 static int
2091 ctl_be_block_close(struct ctl_be_block_lun *be_lun)
2092 {
2093         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2094         int flags;
2095
2096         if (be_lun->vn) {
2097                 flags = FREAD;
2098                 if ((cbe_lun->flags & CTL_LUN_FLAG_READONLY) == 0)
2099                         flags |= FWRITE;
2100                 (void)vn_close(be_lun->vn, flags, NOCRED, curthread);
2101                 be_lun->vn = NULL;
2102
2103                 switch (be_lun->dev_type) {
2104                 case CTL_BE_BLOCK_DEV:
2105                         break;
2106                 case CTL_BE_BLOCK_FILE:
2107                         if (be_lun->backend.file.cred != NULL) {
2108                                 crfree(be_lun->backend.file.cred);
2109                                 be_lun->backend.file.cred = NULL;
2110                         }
2111                         break;
2112                 case CTL_BE_BLOCK_NONE:
2113                         break;
2114                 default:
2115                         panic("Unexpected backend type %d", be_lun->dev_type);
2116                         break;
2117                 }
2118                 be_lun->dev_type = CTL_BE_BLOCK_NONE;
2119         }
2120         return (0);
2121 }
2122
2123 static int
2124 ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
2125 {
2126         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2127         struct nameidata nd;
2128         char            *value;
2129         int              error, flags;
2130
2131         error = 0;
2132         if (rootvnode == NULL) {
2133                 snprintf(req->error_str, sizeof(req->error_str),
2134                          "Root filesystem is not mounted");
2135                 return (1);
2136         }
2137         pwd_ensure_dirs();
2138
2139         value = ctl_get_opt(&cbe_lun->options, "file");
2140         if (value == NULL) {
2141                 snprintf(req->error_str, sizeof(req->error_str),
2142                          "no file argument specified");
2143                 return (1);
2144         }
2145         free(be_lun->dev_path, M_CTLBLK);
2146         be_lun->dev_path = strdup(value, M_CTLBLK);
2147
2148         flags = FREAD;
2149         value = ctl_get_opt(&cbe_lun->options, "readonly");
2150         if (value != NULL) {
2151                 if (strcmp(value, "on") != 0)
2152                         flags |= FWRITE;
2153         } else if (cbe_lun->lun_type == T_DIRECT)
2154                 flags |= FWRITE;
2155
2156 again:
2157         NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread);
2158         error = vn_open(&nd, &flags, 0, NULL);
2159         if ((error == EROFS || error == EACCES) && (flags & FWRITE)) {
2160                 flags &= ~FWRITE;
2161                 goto again;
2162         }
2163         if (error) {
2164                 /*
2165                  * This is the only reasonable guess we can make as far as
2166                  * path if the user doesn't give us a fully qualified path.
2167                  * If they want to specify a file, they need to specify the
2168                  * full path.
2169                  */
2170                 if (be_lun->dev_path[0] != '/') {
2171                         char *dev_name;
2172
2173                         asprintf(&dev_name, M_CTLBLK, "/dev/%s",
2174                                 be_lun->dev_path);
2175                         free(be_lun->dev_path, M_CTLBLK);
2176                         be_lun->dev_path = dev_name;
2177                         goto again;
2178                 }
2179                 snprintf(req->error_str, sizeof(req->error_str),
2180                     "error opening %s: %d", be_lun->dev_path, error);
2181                 return (error);
2182         }
2183         if (flags & FWRITE)
2184                 cbe_lun->flags &= ~CTL_LUN_FLAG_READONLY;
2185         else
2186                 cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
2187
2188         NDFREE(&nd, NDF_ONLY_PNBUF);
2189         be_lun->vn = nd.ni_vp;
2190
2191         /* We only support disks and files. */
2192         if (vn_isdisk(be_lun->vn, &error)) {
2193                 error = ctl_be_block_open_dev(be_lun, req);
2194         } else if (be_lun->vn->v_type == VREG) {
2195                 error = ctl_be_block_open_file(be_lun, req);
2196         } else {
2197                 error = EINVAL;
2198                 snprintf(req->error_str, sizeof(req->error_str),
2199                          "%s is not a disk or plain file", be_lun->dev_path);
2200         }
2201         VOP_UNLOCK(be_lun->vn, 0);
2202
2203         if (error != 0)
2204                 ctl_be_block_close(be_lun);
2205         cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
2206         if (be_lun->dispatch != ctl_be_block_dispatch_dev)
2207                 cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
2208         value = ctl_get_opt(&cbe_lun->options, "serseq");
2209         if (value != NULL && strcmp(value, "on") == 0)
2210                 cbe_lun->serseq = CTL_LUN_SERSEQ_ON;
2211         else if (value != NULL && strcmp(value, "read") == 0)
2212                 cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
2213         else if (value != NULL && strcmp(value, "off") == 0)
2214                 cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
2215         return (0);
2216 }
2217
2218 static int
2219 ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2220 {
2221         struct ctl_be_lun *cbe_lun;
2222         struct ctl_be_block_lun *be_lun;
2223         struct ctl_lun_create_params *params;
2224         char num_thread_str[16];
2225         char tmpstr[32];
2226         char *value;
2227         int retval, num_threads;
2228         int tmp_num_threads;
2229
2230         params = &req->reqdata.create;
2231         retval = 0;
2232         req->status = CTL_LUN_OK;
2233
2234         be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);
2235         cbe_lun = &be_lun->cbe_lun;
2236         cbe_lun->be_lun = be_lun;
2237         be_lun->params = req->reqdata.create;
2238         be_lun->softc = softc;
2239         STAILQ_INIT(&be_lun->input_queue);
2240         STAILQ_INIT(&be_lun->config_read_queue);
2241         STAILQ_INIT(&be_lun->config_write_queue);
2242         STAILQ_INIT(&be_lun->datamove_queue);
2243         sprintf(be_lun->lunname, "cblk%d", softc->num_luns);
2244         mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF);
2245         mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF);
2246         ctl_init_opts(&cbe_lun->options,
2247             req->num_be_args, req->kern_be_args);
2248         be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG,
2249             NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
2250         if (be_lun->lun_zone == NULL) {
2251                 snprintf(req->error_str, sizeof(req->error_str),
2252                          "error allocating UMA zone");
2253                 goto bailout_error;
2254         }
2255
2256         if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
2257                 cbe_lun->lun_type = params->device_type;
2258         else
2259                 cbe_lun->lun_type = T_DIRECT;
2260         be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED;
2261         cbe_lun->flags = 0;
2262         value = ctl_get_opt(&cbe_lun->options, "ha_role");
2263         if (value != NULL) {
2264                 if (strcmp(value, "primary") == 0)
2265                         cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2266         } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
2267                 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2268
2269         if (cbe_lun->lun_type == T_DIRECT ||
2270             cbe_lun->lun_type == T_CDROM) {
2271                 be_lun->size_bytes = params->lun_size_bytes;
2272                 if (params->blocksize_bytes != 0)
2273                         cbe_lun->blocksize = params->blocksize_bytes;
2274                 else if (cbe_lun->lun_type == T_CDROM)
2275                         cbe_lun->blocksize = 2048;
2276                 else
2277                         cbe_lun->blocksize = 512;
2278                 be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
2279                 cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
2280                     0 : (be_lun->size_blocks - 1);
2281
2282                 if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
2283                     control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
2284                         retval = ctl_be_block_open(be_lun, req);
2285                         if (retval != 0) {
2286                                 retval = 0;
2287                                 req->status = CTL_LUN_WARNING;
2288                         }
2289                 }
2290                 num_threads = cbb_num_threads;
2291         } else {
2292                 num_threads = 1;
2293         }
2294
2295         value = ctl_get_opt(&cbe_lun->options, "num_threads");
2296         if (value != NULL) {
2297                 tmp_num_threads = strtol(value, NULL, 0);
2298
2299                 /*
2300                  * We don't let the user specify less than one
2301                  * thread, but hope he's clueful enough not to
2302                  * specify 1000 threads.
2303                  */
2304                 if (tmp_num_threads < 1) {
2305                         snprintf(req->error_str, sizeof(req->error_str),
2306                                  "invalid number of threads %s",
2307                                  num_thread_str);
2308                         goto bailout_error;
2309                 }
2310                 num_threads = tmp_num_threads;
2311         }
2312
2313         if (be_lun->vn == NULL)
2314                 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2315         /* Tell the user the blocksize we ended up using */
2316         params->lun_size_bytes = be_lun->size_bytes;
2317         params->blocksize_bytes = cbe_lun->blocksize;
2318         if (params->flags & CTL_LUN_FLAG_ID_REQ) {
2319                 cbe_lun->req_lun_id = params->req_lun_id;
2320                 cbe_lun->flags |= CTL_LUN_FLAG_ID_REQ;
2321         } else
2322                 cbe_lun->req_lun_id = 0;
2323
2324         cbe_lun->lun_shutdown = ctl_be_block_lun_shutdown;
2325         cbe_lun->lun_config_status = ctl_be_block_lun_config_status;
2326         cbe_lun->be = &ctl_be_block_driver;
2327
2328         if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) {
2329                 snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%04d",
2330                          softc->num_luns);
2331                 strncpy((char *)cbe_lun->serial_num, tmpstr,
2332                         MIN(sizeof(cbe_lun->serial_num), sizeof(tmpstr)));
2333
2334                 /* Tell the user what we used for a serial number */
2335                 strncpy((char *)params->serial_num, tmpstr,
2336                         MIN(sizeof(params->serial_num), sizeof(tmpstr)));
2337         } else {
2338                 strncpy((char *)cbe_lun->serial_num, params->serial_num,
2339                         MIN(sizeof(cbe_lun->serial_num),
2340                         sizeof(params->serial_num)));
2341         }
2342         if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) {
2343                 snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%04d", softc->num_luns);
2344                 strncpy((char *)cbe_lun->device_id, tmpstr,
2345                         MIN(sizeof(cbe_lun->device_id), sizeof(tmpstr)));
2346
2347                 /* Tell the user what we used for a device ID */
2348                 strncpy((char *)params->device_id, tmpstr,
2349                         MIN(sizeof(params->device_id), sizeof(tmpstr)));
2350         } else {
2351                 strncpy((char *)cbe_lun->device_id, params->device_id,
2352                         MIN(sizeof(cbe_lun->device_id),
2353                             sizeof(params->device_id)));
2354         }
2355
2356         TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);
2357
2358         be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK,
2359             taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);
2360
2361         if (be_lun->io_taskqueue == NULL) {
2362                 snprintf(req->error_str, sizeof(req->error_str),
2363                          "unable to create taskqueue");
2364                 goto bailout_error;
2365         }
2366
2367         /*
2368          * Note that we start the same number of threads by default for
2369          * both the file case and the block device case.  For the file
2370          * case, we need multiple threads to allow concurrency, because the
2371          * vnode interface is designed to be a blocking interface.  For the
2372          * block device case, ZFS zvols at least will block the caller's
2373          * context in many instances, and so we need multiple threads to
2374          * overcome that problem.  Other block devices don't need as many
2375          * threads, but they shouldn't cause too many problems.
2376          *
2377          * If the user wants to just have a single thread for a block
2378          * device, he can specify that when the LUN is created, or change
2379          * the tunable/sysctl to alter the default number of threads.
2380          */
2381         retval = taskqueue_start_threads(&be_lun->io_taskqueue,
2382                                          /*num threads*/num_threads,
2383                                          /*priority*/PWAIT,
2384                                          /*thread name*/
2385                                          "%s taskq", be_lun->lunname);
2386
2387         if (retval != 0)
2388                 goto bailout_error;
2389
2390         be_lun->num_threads = num_threads;
2391
2392         mtx_lock(&softc->lock);
2393         softc->num_luns++;
2394         STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links);
2395
2396         mtx_unlock(&softc->lock);
2397
2398         retval = ctl_add_lun(&be_lun->cbe_lun);
2399         if (retval != 0) {
2400                 mtx_lock(&softc->lock);
2401                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2402                               links);
2403                 softc->num_luns--;
2404                 mtx_unlock(&softc->lock);
2405                 snprintf(req->error_str, sizeof(req->error_str),
2406                          "ctl_add_lun() returned error %d, see dmesg for "
2407                          "details", retval);
2408                 retval = 0;
2409                 goto bailout_error;
2410         }
2411
2412         mtx_lock(&softc->lock);
2413
2414         /*
2415          * Tell the config_status routine that we're waiting so it won't
2416          * clean up the LUN in the event of an error.
2417          */
2418         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2419
2420         while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {
2421                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2422                 if (retval == EINTR)
2423                         break;
2424         }
2425         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2426
2427         if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) {
2428                 snprintf(req->error_str, sizeof(req->error_str),
2429                          "LUN configuration error, see dmesg for details");
2430                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2431                               links);
2432                 softc->num_luns--;
2433                 mtx_unlock(&softc->lock);
2434                 goto bailout_error;
2435         } else {
2436                 params->req_lun_id = cbe_lun->lun_id;
2437         }
2438
2439         mtx_unlock(&softc->lock);
2440
2441         be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id,
2442                                                cbe_lun->blocksize,
2443                                                DEVSTAT_ALL_SUPPORTED,
2444                                                cbe_lun->lun_type
2445                                                | DEVSTAT_TYPE_IF_OTHER,
2446                                                DEVSTAT_PRIORITY_OTHER);
2447
2448         return (retval);
2449
2450 bailout_error:
2451         req->status = CTL_LUN_ERROR;
2452
2453         if (be_lun->io_taskqueue != NULL)
2454                 taskqueue_free(be_lun->io_taskqueue);
2455         ctl_be_block_close(be_lun);
2456         if (be_lun->dev_path != NULL)
2457                 free(be_lun->dev_path, M_CTLBLK);
2458         if (be_lun->lun_zone != NULL)
2459                 uma_zdestroy(be_lun->lun_zone);
2460         ctl_free_opts(&cbe_lun->options);
2461         mtx_destroy(&be_lun->queue_lock);
2462         mtx_destroy(&be_lun->io_lock);
2463         free(be_lun, M_CTLBLK);
2464
2465         return (retval);
2466 }
2467
2468 static int
2469 ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2470 {
2471         struct ctl_lun_rm_params *params;
2472         struct ctl_be_block_lun *be_lun;
2473         struct ctl_be_lun *cbe_lun;
2474         int retval;
2475
2476         params = &req->reqdata.rm;
2477
2478         mtx_lock(&softc->lock);
2479         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2480                 if (be_lun->cbe_lun.lun_id == params->lun_id)
2481                         break;
2482         }
2483         mtx_unlock(&softc->lock);
2484         if (be_lun == NULL) {
2485                 snprintf(req->error_str, sizeof(req->error_str),
2486                          "LUN %u is not managed by the block backend",
2487                          params->lun_id);
2488                 goto bailout_error;
2489         }
2490         cbe_lun = &be_lun->cbe_lun;
2491
2492         retval = ctl_disable_lun(cbe_lun);
2493         if (retval != 0) {
2494                 snprintf(req->error_str, sizeof(req->error_str),
2495                          "error %d returned from ctl_disable_lun() for "
2496                          "LUN %d", retval, params->lun_id);
2497                 goto bailout_error;
2498         }
2499
2500         if (be_lun->vn != NULL) {
2501                 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2502                 ctl_lun_no_media(cbe_lun);
2503                 taskqueue_drain_all(be_lun->io_taskqueue);
2504                 ctl_be_block_close(be_lun);
2505         }
2506
2507         retval = ctl_invalidate_lun(cbe_lun);
2508         if (retval != 0) {
2509                 snprintf(req->error_str, sizeof(req->error_str),
2510                          "error %d returned from ctl_invalidate_lun() for "
2511                          "LUN %d", retval, params->lun_id);
2512                 goto bailout_error;
2513         }
2514
2515         mtx_lock(&softc->lock);
2516         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2517         while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2518                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2519                 if (retval == EINTR)
2520                         break;
2521         }
2522         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2523
2524         if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2525                 snprintf(req->error_str, sizeof(req->error_str),
2526                          "interrupted waiting for LUN to be freed");
2527                 mtx_unlock(&softc->lock);
2528                 goto bailout_error;
2529         }
2530
2531         STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links);
2532
2533         softc->num_luns--;
2534         mtx_unlock(&softc->lock);
2535
2536         taskqueue_drain_all(be_lun->io_taskqueue);
2537         taskqueue_free(be_lun->io_taskqueue);
2538
2539         if (be_lun->disk_stats != NULL)
2540                 devstat_remove_entry(be_lun->disk_stats);
2541
2542         uma_zdestroy(be_lun->lun_zone);
2543
2544         ctl_free_opts(&cbe_lun->options);
2545         free(be_lun->dev_path, M_CTLBLK);
2546         mtx_destroy(&be_lun->queue_lock);
2547         mtx_destroy(&be_lun->io_lock);
2548         free(be_lun, M_CTLBLK);
2549
2550         req->status = CTL_LUN_OK;
2551         return (0);
2552
2553 bailout_error:
2554         req->status = CTL_LUN_ERROR;
2555         return (0);
2556 }
2557
2558 static int
2559 ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2560 {
2561         struct ctl_lun_modify_params *params;
2562         struct ctl_be_block_lun *be_lun;
2563         struct ctl_be_lun *cbe_lun;
2564         char *value;
2565         uint64_t oldsize;
2566         int error, wasprim;
2567
2568         params = &req->reqdata.modify;
2569
2570         mtx_lock(&softc->lock);
2571         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2572                 if (be_lun->cbe_lun.lun_id == params->lun_id)
2573                         break;
2574         }
2575         mtx_unlock(&softc->lock);
2576         if (be_lun == NULL) {
2577                 snprintf(req->error_str, sizeof(req->error_str),
2578                          "LUN %u is not managed by the block backend",
2579                          params->lun_id);
2580                 goto bailout_error;
2581         }
2582         cbe_lun = &be_lun->cbe_lun;
2583
2584         if (params->lun_size_bytes != 0)
2585                 be_lun->params.lun_size_bytes = params->lun_size_bytes;
2586         ctl_update_opts(&cbe_lun->options, req->num_be_args, req->kern_be_args);
2587
2588         wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY);
2589         value = ctl_get_opt(&cbe_lun->options, "ha_role");
2590         if (value != NULL) {
2591                 if (strcmp(value, "primary") == 0)
2592                         cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2593                 else
2594                         cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
2595         } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
2596                 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2597         else
2598                 cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
2599         if (wasprim != (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)) {
2600                 if (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)
2601                         ctl_lun_primary(cbe_lun);
2602                 else
2603                         ctl_lun_secondary(cbe_lun);
2604         }
2605
2606         oldsize = be_lun->size_blocks;
2607         if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
2608             control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
2609                 if (be_lun->vn == NULL)
2610                         error = ctl_be_block_open(be_lun, req);
2611                 else if (vn_isdisk(be_lun->vn, &error))
2612                         error = ctl_be_block_open_dev(be_lun, req);
2613                 else if (be_lun->vn->v_type == VREG) {
2614                         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
2615                         error = ctl_be_block_open_file(be_lun, req);
2616                         VOP_UNLOCK(be_lun->vn, 0);
2617                 } else
2618                         error = EINVAL;
2619                 if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) &&
2620                     be_lun->vn != NULL) {
2621                         cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
2622                         ctl_lun_has_media(cbe_lun);
2623                 } else if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) == 0 &&
2624                     be_lun->vn == NULL) {
2625                         cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2626                         ctl_lun_no_media(cbe_lun);
2627                 }
2628                 cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
2629         } else {
2630                 if (be_lun->vn != NULL) {
2631                         cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2632                         ctl_lun_no_media(cbe_lun);
2633                         taskqueue_drain_all(be_lun->io_taskqueue);
2634                         error = ctl_be_block_close(be_lun);
2635                 } else
2636                         error = 0;
2637         }
2638         if (be_lun->size_blocks != oldsize)
2639                 ctl_lun_capacity_changed(cbe_lun);
2640
2641         /* Tell the user the exact size we ended up using */
2642         params->lun_size_bytes = be_lun->size_bytes;
2643
2644         req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK;
2645         return (0);
2646
2647 bailout_error:
2648         req->status = CTL_LUN_ERROR;
2649         return (0);
2650 }
2651
2652 static void
2653 ctl_be_block_lun_shutdown(void *be_lun)
2654 {
2655         struct ctl_be_block_lun *lun = be_lun;
2656         struct ctl_be_block_softc *softc = lun->softc;
2657
2658         mtx_lock(&softc->lock);
2659         lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;
2660         if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2661                 wakeup(lun);
2662         mtx_unlock(&softc->lock);
2663 }
2664
2665 static void
2666 ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status)
2667 {
2668         struct ctl_be_block_lun *lun;
2669         struct ctl_be_block_softc *softc;
2670
2671         lun = (struct ctl_be_block_lun *)be_lun;
2672         softc = lun->softc;
2673
2674         if (status == CTL_LUN_CONFIG_OK) {
2675                 mtx_lock(&softc->lock);
2676                 lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2677                 if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2678                         wakeup(lun);
2679                 mtx_unlock(&softc->lock);
2680
2681                 /*
2682                  * We successfully added the LUN, attempt to enable it.
2683                  */
2684                 if (ctl_enable_lun(&lun->cbe_lun) != 0) {
2685                         printf("%s: ctl_enable_lun() failed!\n", __func__);
2686                         if (ctl_invalidate_lun(&lun->cbe_lun) != 0) {
2687                                 printf("%s: ctl_invalidate_lun() failed!\n",
2688                                        __func__);
2689                         }
2690                 }
2691
2692                 return;
2693         }
2694
2695
2696         mtx_lock(&softc->lock);
2697         lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2698         lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR;
2699         wakeup(lun);
2700         mtx_unlock(&softc->lock);
2701 }
2702
2703
2704 static int
2705 ctl_be_block_config_write(union ctl_io *io)
2706 {
2707         struct ctl_be_block_lun *be_lun;
2708         struct ctl_be_lun *cbe_lun;
2709         int retval;
2710
2711         DPRINTF("entered\n");
2712
2713         cbe_lun = CTL_BACKEND_LUN(io);
2714         be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
2715
2716         retval = 0;
2717         switch (io->scsiio.cdb[0]) {
2718         case SYNCHRONIZE_CACHE:
2719         case SYNCHRONIZE_CACHE_16:
2720         case WRITE_SAME_10:
2721         case WRITE_SAME_16:
2722         case UNMAP:
2723                 /*
2724                  * The upper level CTL code will filter out any CDBs with
2725                  * the immediate bit set and return the proper error.
2726                  *
2727                  * We don't really need to worry about what LBA range the
2728                  * user asked to be synced out.  When they issue a sync
2729                  * cache command, we'll sync out the whole thing.
2730                  */
2731                 mtx_lock(&be_lun->queue_lock);
2732                 STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,
2733                                    links);
2734                 mtx_unlock(&be_lun->queue_lock);
2735                 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
2736                 break;
2737         case START_STOP_UNIT: {
2738                 struct scsi_start_stop_unit *cdb;
2739                 struct ctl_lun_req req;
2740
2741                 cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
2742                 if ((cdb->how & SSS_PC_MASK) != 0) {
2743                         ctl_set_success(&io->scsiio);
2744                         ctl_config_write_done(io);
2745                         break;
2746                 }
2747                 if (cdb->how & SSS_START) {
2748                         if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) {
2749                                 retval = ctl_be_block_open(be_lun, &req);
2750                                 cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
2751                                 if (retval == 0) {
2752                                         cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
2753                                         ctl_lun_has_media(cbe_lun);
2754                                 } else {
2755                                         cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2756                                         ctl_lun_no_media(cbe_lun);
2757                                 }
2758                         }
2759                         ctl_start_lun(cbe_lun);
2760                 } else {
2761                         ctl_stop_lun(cbe_lun);
2762                         if (cdb->how & SSS_LOEJ) {
2763                                 cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
2764                                 cbe_lun->flags |= CTL_LUN_FLAG_EJECTED;
2765                                 ctl_lun_ejected(cbe_lun);
2766                                 if (be_lun->vn != NULL)
2767                                         ctl_be_block_close(be_lun);
2768                         }
2769                 }
2770
2771                 ctl_set_success(&io->scsiio);
2772                 ctl_config_write_done(io);
2773                 break;
2774         }
2775         case PREVENT_ALLOW:
2776                 ctl_set_success(&io->scsiio);
2777                 ctl_config_write_done(io);
2778                 break;
2779         default:
2780                 ctl_set_invalid_opcode(&io->scsiio);
2781                 ctl_config_write_done(io);
2782                 retval = CTL_RETVAL_COMPLETE;
2783                 break;
2784         }
2785
2786         return (retval);
2787 }
2788
2789 static int
2790 ctl_be_block_config_read(union ctl_io *io)
2791 {
2792         struct ctl_be_block_lun *be_lun;
2793         struct ctl_be_lun *cbe_lun;
2794         int retval = 0;
2795
2796         DPRINTF("entered\n");
2797
2798         cbe_lun = CTL_BACKEND_LUN(io);
2799         be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
2800
2801         switch (io->scsiio.cdb[0]) {
2802         case SERVICE_ACTION_IN:
2803                 if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) {
2804                         mtx_lock(&be_lun->queue_lock);
2805                         STAILQ_INSERT_TAIL(&be_lun->config_read_queue,
2806                             &io->io_hdr, links);
2807                         mtx_unlock(&be_lun->queue_lock);
2808                         taskqueue_enqueue(be_lun->io_taskqueue,
2809                             &be_lun->io_task);
2810                         retval = CTL_RETVAL_QUEUED;
2811                         break;
2812                 }
2813                 ctl_set_invalid_field(&io->scsiio,
2814                                       /*sks_valid*/ 1,
2815                                       /*command*/ 1,
2816                                       /*field*/ 1,
2817                                       /*bit_valid*/ 1,
2818                                       /*bit*/ 4);
2819                 ctl_config_read_done(io);
2820                 retval = CTL_RETVAL_COMPLETE;
2821                 break;
2822         default:
2823                 ctl_set_invalid_opcode(&io->scsiio);
2824                 ctl_config_read_done(io);
2825                 retval = CTL_RETVAL_COMPLETE;
2826                 break;
2827         }
2828
2829         return (retval);
2830 }
2831
2832 static int
2833 ctl_be_block_lun_info(void *be_lun, struct sbuf *sb)
2834 {
2835         struct ctl_be_block_lun *lun;
2836         int retval;
2837
2838         lun = (struct ctl_be_block_lun *)be_lun;
2839
2840         retval = sbuf_printf(sb, "\t<num_threads>");
2841         if (retval != 0)
2842                 goto bailout;
2843         retval = sbuf_printf(sb, "%d", lun->num_threads);
2844         if (retval != 0)
2845                 goto bailout;
2846         retval = sbuf_printf(sb, "</num_threads>\n");
2847
2848 bailout:
2849         return (retval);
2850 }
2851
2852 static uint64_t
2853 ctl_be_block_lun_attr(void *be_lun, const char *attrname)
2854 {
2855         struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun;
2856
2857         if (lun->getattr == NULL)
2858                 return (UINT64_MAX);
2859         return (lun->getattr(lun, attrname));
2860 }
2861
2862 static int
2863 ctl_be_block_init(void)
2864 {
2865         struct ctl_be_block_softc *softc = &backend_block_softc;
2866
2867         mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF);
2868         softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io),
2869             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
2870         STAILQ_INIT(&softc->lun_list);
2871         return (0);
2872 }
2873
2874
2875 static int
2876 ctl_be_block_shutdown(void)
2877 {
2878         struct ctl_be_block_softc *softc = &backend_block_softc;
2879         struct ctl_be_block_lun *lun, *next_lun;
2880
2881         mtx_lock(&softc->lock);
2882         STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) {
2883                 /*
2884                  * Drop our lock here.  Since ctl_invalidate_lun() can call
2885                  * back into us, this could potentially lead to a recursive
2886                  * lock of the same mutex, which would cause a hang.
2887                  */
2888                 mtx_unlock(&softc->lock);
2889                 ctl_disable_lun(&lun->cbe_lun);
2890                 ctl_invalidate_lun(&lun->cbe_lun);
2891                 mtx_lock(&softc->lock);
2892         }
2893         mtx_unlock(&softc->lock);
2894
2895         uma_zdestroy(softc->beio_zone);
2896         mtx_destroy(&softc->lock);
2897         return (0);
2898 }