sys/cam/ctl/ctl_backend_block.c

   1 /*-
   2  * Copyright (c) 2003 Silicon Graphics International Corp.
   3  * Copyright (c) 2009-2011 Spectra Logic Corporation
   4  * Copyright (c) 2012 The FreeBSD Foundation
   5  * All rights reserved.
   6  *
   7  * Portions of this software were developed by Edward Tomasz Napierala
   8  * under sponsorship from the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions, and the following disclaimer,
  15  *    without modification.
  16  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  17  *    substantially similar to the "NO WARRANTY" disclaimer below
  18  *    ("Disclaimer") and any redistribution must be conditioned upon
  19  *    including a substantially similar Disclaimer requirement for further
  20  *    binary redistribution.
  21  *
  22  * NO WARRANTY
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  32  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  33  * POSSIBILITY OF SUCH DAMAGES.
  34  *
  35  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
  36  */
  37 /*
  38  * CAM Target Layer driver backend for block devices.
  39  *
  40  * Author: Ken Merry <ken@FreeBSD.org>
  41  */
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 #include <sys/param.h>
  46 #include <sys/systm.h>
  47 #include <sys/kernel.h>
  48 #include <sys/types.h>
  49 #include <sys/kthread.h>
  50 #include <sys/bio.h>
  51 #include <sys/fcntl.h>
  52 #include <sys/limits.h>
  53 #include <sys/lock.h>
  54 #include <sys/mutex.h>
  55 #include <sys/condvar.h>
  56 #include <sys/malloc.h>
  57 #include <sys/conf.h>
  58 #include <sys/ioccom.h>
  59 #include <sys/queue.h>
  60 #include <sys/sbuf.h>
  61 #include <sys/endian.h>
  62 #include <sys/uio.h>
  63 #include <sys/buf.h>
  64 #include <sys/taskqueue.h>
  65 #include <sys/vnode.h>
  66 #include <sys/namei.h>
  67 #include <sys/mount.h>
  68 #include <sys/disk.h>
  69 #include <sys/fcntl.h>
  70 #include <sys/filedesc.h>
  71 #include <sys/filio.h>
  72 #include <sys/proc.h>
  73 #include <sys/pcpu.h>
  74 #include <sys/module.h>
  75 #include <sys/sdt.h>
  76 #include <sys/devicestat.h>
  77 #include <sys/sysctl.h>
  78
  79 #include <geom/geom.h>
  80
  81 #include <cam/cam.h>
  82 #include <cam/scsi/scsi_all.h>
  83 #include <cam/scsi/scsi_da.h>
  84 #include <cam/ctl/ctl_io.h>
  85 #include <cam/ctl/ctl.h>
  86 #include <cam/ctl/ctl_backend.h>
  87 #include <cam/ctl/ctl_ioctl.h>
  88 #include <cam/ctl/ctl_ha.h>
  89 #include <cam/ctl/ctl_scsi_all.h>
  90 #include <cam/ctl/ctl_private.h>
  91 #include <cam/ctl/ctl_error.h>
  92
  93 /*
  94  * The idea here is that we'll allocate enough S/G space to hold a 1MB
  95  * I/O.  If we get an I/O larger than that, we'll split it.
  96  */
  97 #define CTLBLK_HALF_IO_SIZE     (512 * 1024)
  98 #define CTLBLK_MAX_IO_SIZE      (CTLBLK_HALF_IO_SIZE * 2)
  99 #define CTLBLK_MAX_SEG          MAXPHYS
 100 #define CTLBLK_HALF_SEGS        MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1)
 101 #define CTLBLK_MAX_SEGS         (CTLBLK_HALF_SEGS * 2)
 102
 103 #ifdef CTLBLK_DEBUG
 104 #define DPRINTF(fmt, args...) \
 105     printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args)
 106 #else
 107 #define DPRINTF(fmt, args...) do {} while(0)
 108 #endif
 109
 110 #define PRIV(io)        \
 111     ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND])
 112 #define ARGS(io)        \
 113     ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN])
 114
 115 SDT_PROVIDER_DEFINE(cbb);
 116
 117 typedef enum {
 118         CTL_BE_BLOCK_LUN_UNCONFIGURED   = 0x01,
 119         CTL_BE_BLOCK_LUN_CONFIG_ERR     = 0x02,
 120         CTL_BE_BLOCK_LUN_WAITING        = 0x04,
 121 } ctl_be_block_lun_flags;
 122
 123 typedef enum {
 124         CTL_BE_BLOCK_NONE,
 125         CTL_BE_BLOCK_DEV,
 126         CTL_BE_BLOCK_FILE
 127 } ctl_be_block_type;
 128
 129 struct ctl_be_block_filedata {
 130         struct ucred *cred;
 131 };
 132
 133 union ctl_be_block_bedata {
 134         struct ctl_be_block_filedata file;
 135 };
 136
 137 struct ctl_be_block_io;
 138 struct ctl_be_block_lun;
 139
 140 typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
 141                                struct ctl_be_block_io *beio);
 142 typedef uint64_t (*cbb_getattr_t)(struct ctl_be_block_lun *be_lun,
 143                                   const char *attrname);
 144
 145 /*
 146  * Backend LUN structure.  There is a 1:1 mapping between a block device
 147  * and a backend block LUN, and between a backend block LUN and a CTL LUN.
 148  */
 149 struct ctl_be_block_lun {
 150         struct ctl_lun_create_params params;
 151         char lunname[32];
 152         char *dev_path;
 153         ctl_be_block_type dev_type;
 154         struct vnode *vn;
 155         union ctl_be_block_bedata backend;
 156         cbb_dispatch_t dispatch;
 157         cbb_dispatch_t lun_flush;
 158         cbb_dispatch_t unmap;
 159         cbb_dispatch_t get_lba_status;
 160         cbb_getattr_t getattr;
 161         uma_zone_t lun_zone;
 162         uint64_t size_blocks;
 163         uint64_t size_bytes;
 164         struct ctl_be_block_softc *softc;
 165         struct devstat *disk_stats;
 166         ctl_be_block_lun_flags flags;
 167         STAILQ_ENTRY(ctl_be_block_lun) links;
 168         struct ctl_be_lun cbe_lun;
 169         struct taskqueue *io_taskqueue;
 170         struct task io_task;
 171         int num_threads;
 172         STAILQ_HEAD(, ctl_io_hdr) input_queue;
 173         STAILQ_HEAD(, ctl_io_hdr) config_read_queue;
 174         STAILQ_HEAD(, ctl_io_hdr) config_write_queue;
 175         STAILQ_HEAD(, ctl_io_hdr) datamove_queue;
 176         struct mtx_padalign io_lock;
 177         struct mtx_padalign queue_lock;
 178 };
 179
 180 /*
 181  * Overall softc structure for the block backend module.
 182  */
 183 struct ctl_be_block_softc {
 184         struct mtx                       lock;
 185         int                              num_luns;
 186         STAILQ_HEAD(, ctl_be_block_lun)  lun_list;
 187 };
 188
 189 static struct ctl_be_block_softc backend_block_softc;
 190
 191 /*
 192  * Per-I/O information.
 193  */
 194 struct ctl_be_block_io {
 195         union ctl_io                    *io;
 196         struct ctl_sg_entry             sg_segs[CTLBLK_MAX_SEGS];
 197         struct iovec                    xiovecs[CTLBLK_MAX_SEGS];
 198         int                             bio_cmd;
 199         int                             num_segs;
 200         int                             num_bios_sent;
 201         int                             num_bios_done;
 202         int                             send_complete;
 203         int                             num_errors;
 204         struct bintime                  ds_t0;
 205         devstat_tag_type                ds_tag_type;
 206         devstat_trans_flags             ds_trans_type;
 207         uint64_t                        io_len;
 208         uint64_t                        io_offset;
 209         int                             io_arg;
 210         struct ctl_be_block_softc       *softc;
 211         struct ctl_be_block_lun         *lun;
 212         void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */
 213 };
 214
 215 extern struct ctl_softc *control_softc;
 216
 217 static int cbb_num_threads = 14;
 218 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0,
 219             "CAM Target Layer Block Backend");
 220 SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RWTUN,
 221            &cbb_num_threads, 0, "Number of threads per backing file");
 222
 223 static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
 224 static void ctl_free_beio(struct ctl_be_block_io *beio);
 225 static void ctl_complete_beio(struct ctl_be_block_io *beio);
 226 static int ctl_be_block_move_done(union ctl_io *io);
 227 static void ctl_be_block_biodone(struct bio *bio);
 228 static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 229                                     struct ctl_be_block_io *beio);
 230 static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 231                                        struct ctl_be_block_io *beio);
 232 static void ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
 233                                   struct ctl_be_block_io *beio);
 234 static uint64_t ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun,
 235                                          const char *attrname);
 236 static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 237                                    struct ctl_be_block_io *beio);
 238 static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
 239                                    struct ctl_be_block_io *beio);
 240 static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
 241                                       struct ctl_be_block_io *beio);
 242 static uint64_t ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun,
 243                                          const char *attrname);
 244 static void ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
 245                                     union ctl_io *io);
 246 static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
 247                                     union ctl_io *io);
 248 static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 249                                   union ctl_io *io);
 250 static void ctl_be_block_worker(void *context, int pending);
 251 static int ctl_be_block_submit(union ctl_io *io);
 252 static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
 253                                    int flag, struct thread *td);
 254 static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
 255                                   struct ctl_lun_req *req);
 256 static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
 257                                  struct ctl_lun_req *req);
 258 static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
 259 static int ctl_be_block_open(struct ctl_be_block_softc *softc,
 260                              struct ctl_be_block_lun *be_lun,
 261                              struct ctl_lun_req *req);
 262 static int ctl_be_block_create(struct ctl_be_block_softc *softc,
 263                                struct ctl_lun_req *req);
 264 static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
 265                            struct ctl_lun_req *req);
 266 static int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
 267                                   struct ctl_lun_req *req);
 268 static int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
 269                                  struct ctl_lun_req *req);
 270 static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
 271                            struct ctl_lun_req *req);
 272 static void ctl_be_block_lun_shutdown(void *be_lun);
 273 static void ctl_be_block_lun_config_status(void *be_lun,
 274                                            ctl_lun_config_status status);
 275 static int ctl_be_block_config_write(union ctl_io *io);
 276 static int ctl_be_block_config_read(union ctl_io *io);
 277 static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb);
 278 static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname);
 279 int ctl_be_block_init(void);
 280
 281 static struct ctl_backend_driver ctl_be_block_driver =
 282 {
 283         .name = "block",
 284         .flags = CTL_BE_FLAG_HAS_CONFIG,
 285         .init = ctl_be_block_init,
 286         .data_submit = ctl_be_block_submit,
 287         .data_move_done = ctl_be_block_move_done,
 288         .config_read = ctl_be_block_config_read,
 289         .config_write = ctl_be_block_config_write,
 290         .ioctl = ctl_be_block_ioctl,
 291         .lun_info = ctl_be_block_lun_info,
 292         .lun_attr = ctl_be_block_lun_attr
 293 };
 294
 295 MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend");
 296 CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
 297
 298 static uma_zone_t beio_zone;
 299
 300 static struct ctl_be_block_io *
 301 ctl_alloc_beio(struct ctl_be_block_softc *softc)
 302 {
 303         struct ctl_be_block_io *beio;
 304
 305         beio = uma_zalloc(beio_zone, M_WAITOK | M_ZERO);
 306         beio->softc = softc;
 307         return (beio);
 308 }
 309
 310 static void
 311 ctl_free_beio(struct ctl_be_block_io *beio)
 312 {
 313         int duplicate_free;
 314         int i;
 315
 316         duplicate_free = 0;
 317
 318         for (i = 0; i < beio->num_segs; i++) {
 319                 if (beio->sg_segs[i].addr == NULL)
 320                         duplicate_free++;
 321
 322                 uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr);
 323                 beio->sg_segs[i].addr = NULL;
 324
 325                 /* For compare we had two equal S/G lists. */
 326                 if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) {
 327                         uma_zfree(beio->lun->lun_zone,
 328                             beio->sg_segs[i + CTLBLK_HALF_SEGS].addr);
 329                         beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL;
 330                 }
 331         }
 332
 333         if (duplicate_free > 0) {
 334                 printf("%s: %d duplicate frees out of %d segments\n", __func__,
 335                        duplicate_free, beio->num_segs);
 336         }
 337
 338         uma_zfree(beio_zone, beio);
 339 }
 340
 341 static void
 342 ctl_complete_beio(struct ctl_be_block_io *beio)
 343 {
 344         union ctl_io *io = beio->io;
 345
 346         if (beio->beio_cont != NULL) {
 347                 beio->beio_cont(beio);
 348         } else {
 349                 ctl_free_beio(beio);
 350                 ctl_data_submit_done(io);
 351         }
 352 }
 353
 354 static int
 355 ctl_be_block_move_done(union ctl_io *io)
 356 {
 357         struct ctl_be_block_io *beio;
 358         struct ctl_be_block_lun *be_lun;
 359         struct ctl_lba_len_flags *lbalen;
 360 #ifdef CTL_TIME_IO
 361         struct bintime cur_bt;
 362 #endif
 363         int i;
 364
 365         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 366         be_lun = beio->lun;
 367
 368         DPRINTF("entered\n");
 369
 370 #ifdef CTL_TIME_IO
 371         getbintime(&cur_bt);
 372         bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
 373         bintime_add(&io->io_hdr.dma_bt, &cur_bt);
 374         io->io_hdr.num_dmas++;
 375 #endif
 376         io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 377
 378         /*
 379          * We set status at this point for read commands, and write
 380          * commands with errors.
 381          */
 382         if (io->io_hdr.flags & CTL_FLAG_ABORT) {
 383                 ;
 384         } else if ((io->io_hdr.port_status == 0) &&
 385             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
 386                 lbalen = ARGS(beio->io);
 387                 if (lbalen->flags & CTL_LLF_READ) {
 388                         ctl_set_success(&io->scsiio);
 389                 } else if (lbalen->flags & CTL_LLF_COMPARE) {
 390                         /* We have two data blocks ready for comparison. */
 391                         for (i = 0; i < beio->num_segs; i++) {
 392                                 if (memcmp(beio->sg_segs[i].addr,
 393                                     beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
 394                                     beio->sg_segs[i].len) != 0)
 395                                         break;
 396                         }
 397                         if (i < beio->num_segs)
 398                                 ctl_set_sense(&io->scsiio,
 399                                     /*current_error*/ 1,
 400                                     /*sense_key*/ SSD_KEY_MISCOMPARE,
 401                                     /*asc*/ 0x1D,
 402                                     /*ascq*/ 0x00,
 403                                     SSD_ELEM_NONE);
 404                         else
 405                                 ctl_set_success(&io->scsiio);
 406                 }
 407         } else if ((io->io_hdr.port_status != 0) &&
 408             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
 409              (io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)) {
 410                 /*
 411                  * For hardware error sense keys, the sense key
 412                  * specific value is defined to be a retry count,
 413                  * but we use it to pass back an internal FETD
 414                  * error code.  XXX KDM  Hopefully the FETD is only
 415                  * using 16 bits for an error code, since that's
 416                  * all the space we have in the sks field.
 417                  */
 418                 ctl_set_internal_failure(&io->scsiio,
 419                                          /*sks_valid*/ 1,
 420                                          /*retry_count*/
 421                                          io->io_hdr.port_status);
 422         }
 423
 424         /*
 425          * If this is a read, or a write with errors, it is done.
 426          */
 427         if ((beio->bio_cmd == BIO_READ)
 428          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)
 429          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) {
 430                 ctl_complete_beio(beio);
 431                 return (0);
 432         }
 433
 434         /*
 435          * At this point, we have a write and the DMA completed
 436          * successfully.  We now have to queue it to the task queue to
 437          * execute the backend I/O.  That is because we do blocking
 438          * memory allocations, and in the file backing case, blocking I/O.
 439          * This move done routine is generally called in the SIM's
 440          * interrupt context, and therefore we cannot block.
 441          */
 442         mtx_lock(&be_lun->queue_lock);
 443         /*
 444          * XXX KDM make sure that links is okay to use at this point.
 445          * Otherwise, we either need to add another field to ctl_io_hdr,
 446          * or deal with resource allocation here.
 447          */
 448         STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
 449         mtx_unlock(&be_lun->queue_lock);
 450
 451         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
 452
 453         return (0);
 454 }
 455
 456 static void
 457 ctl_be_block_biodone(struct bio *bio)
 458 {
 459         struct ctl_be_block_io *beio;
 460         struct ctl_be_block_lun *be_lun;
 461         union ctl_io *io;
 462         int error;
 463
 464         beio = bio->bio_caller1;
 465         be_lun = beio->lun;
 466         io = beio->io;
 467
 468         DPRINTF("entered\n");
 469
 470         error = bio->bio_error;
 471         mtx_lock(&be_lun->io_lock);
 472         if (error != 0)
 473                 beio->num_errors++;
 474
 475         beio->num_bios_done++;
 476
 477         /*
 478          * XXX KDM will this cause WITNESS to complain?  Holding a lock
 479          * during the free might cause it to complain.
 480          */
 481         g_destroy_bio(bio);
 482
 483         /*
 484          * If the send complete bit isn't set, or we aren't the last I/O to
 485          * complete, then we're done.
 486          */
 487         if ((beio->send_complete == 0)
 488          || (beio->num_bios_done < beio->num_bios_sent)) {
 489                 mtx_unlock(&be_lun->io_lock);
 490                 return;
 491         }
 492
 493         /*
 494          * At this point, we've verified that we are the last I/O to
 495          * complete, so it's safe to drop the lock.
 496          */
 497         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 498             beio->ds_tag_type, beio->ds_trans_type,
 499             /*now*/ NULL, /*then*/&beio->ds_t0);
 500         mtx_unlock(&be_lun->io_lock);
 501
 502         /*
 503          * If there are any errors from the backing device, we fail the
 504          * entire I/O with a medium error.
 505          */
 506         if (beio->num_errors > 0) {
 507                 if (error == EOPNOTSUPP) {
 508                         ctl_set_invalid_opcode(&io->scsiio);
 509                 } else if (error == ENOSPC || error == EDQUOT) {
 510                         ctl_set_space_alloc_fail(&io->scsiio);
 511                 } else if (beio->bio_cmd == BIO_FLUSH) {
 512                         /* XXX KDM is there is a better error here? */
 513                         ctl_set_internal_failure(&io->scsiio,
 514                                                  /*sks_valid*/ 1,
 515                                                  /*retry_count*/ 0xbad2);
 516                 } else
 517                         ctl_set_medium_error(&io->scsiio);
 518                 ctl_complete_beio(beio);
 519                 return;
 520         }
 521
 522         /*
 523          * If this is a write, a flush, a delete or verify, we're all done.
 524          * If this is a read, we can now send the data to the user.
 525          */
 526         if ((beio->bio_cmd == BIO_WRITE)
 527          || (beio->bio_cmd == BIO_FLUSH)
 528          || (beio->bio_cmd == BIO_DELETE)
 529          || (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 530                 ctl_set_success(&io->scsiio);
 531                 ctl_complete_beio(beio);
 532         } else {
 533                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 534                     beio->beio_cont == NULL)
 535                         ctl_set_success(&io->scsiio);
 536 #ifdef CTL_TIME_IO
 537                 getbintime(&io->io_hdr.dma_start_bt);
 538 #endif
 539                 ctl_datamove(io);
 540         }
 541 }
 542
 543 static void
 544 ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 545                         struct ctl_be_block_io *beio)
 546 {
 547         union ctl_io *io = beio->io;
 548         struct mount *mountpoint;
 549         int error, lock_flags;
 550
 551         DPRINTF("entered\n");
 552
 553         binuptime(&beio->ds_t0);
 554         mtx_lock(&be_lun->io_lock);
 555         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 556         mtx_unlock(&be_lun->io_lock);
 557
 558         (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 559
 560         if (MNT_SHARED_WRITES(mountpoint)
 561          || ((mountpoint == NULL)
 562           && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 563                 lock_flags = LK_SHARED;
 564         else
 565                 lock_flags = LK_EXCLUSIVE;
 566
 567         vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 568
 569         error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT,
 570             curthread);
 571         VOP_UNLOCK(be_lun->vn, 0);
 572
 573         vn_finished_write(mountpoint);
 574
 575         mtx_lock(&be_lun->io_lock);
 576         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 577             beio->ds_tag_type, beio->ds_trans_type,
 578             /*now*/ NULL, /*then*/&beio->ds_t0);
 579         mtx_unlock(&be_lun->io_lock);
 580
 581         if (error == 0)
 582                 ctl_set_success(&io->scsiio);
 583         else {
 584                 /* XXX KDM is there is a better error here? */
 585                 ctl_set_internal_failure(&io->scsiio,
 586                                          /*sks_valid*/ 1,
 587                                          /*retry_count*/ 0xbad1);
 588         }
 589
 590         ctl_complete_beio(beio);
 591 }
 592
 593 SDT_PROBE_DEFINE1(cbb, kernel, read, file_start, "uint64_t");
 594 SDT_PROBE_DEFINE1(cbb, kernel, write, file_start, "uint64_t");
 595 SDT_PROBE_DEFINE1(cbb, kernel, read, file_done,"uint64_t");
 596 SDT_PROBE_DEFINE1(cbb, kernel, write, file_done, "uint64_t");
 597
 598 static void
 599 ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 600                            struct ctl_be_block_io *beio)
 601 {
 602         struct ctl_be_block_filedata *file_data;
 603         union ctl_io *io;
 604         struct uio xuio;
 605         struct iovec *xiovec;
 606         int flags;
 607         int error, i;
 608
 609         DPRINTF("entered\n");
 610
 611         file_data = &be_lun->backend.file;
 612         io = beio->io;
 613         flags = 0;
 614         if (ARGS(io)->flags & CTL_LLF_DPO)
 615                 flags |= IO_DIRECT;
 616         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 617                 flags |= IO_SYNC;
 618
 619         bzero(&xuio, sizeof(xuio));
 620         if (beio->bio_cmd == BIO_READ) {
 621                 SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0);
 622                 xuio.uio_rw = UIO_READ;
 623         } else {
 624                 SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0);
 625                 xuio.uio_rw = UIO_WRITE;
 626         }
 627         xuio.uio_offset = beio->io_offset;
 628         xuio.uio_resid = beio->io_len;
 629         xuio.uio_segflg = UIO_SYSSPACE;
 630         xuio.uio_iov = beio->xiovecs;
 631         xuio.uio_iovcnt = beio->num_segs;
 632         xuio.uio_td = curthread;
 633
 634         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 635                 xiovec->iov_base = beio->sg_segs[i].addr;
 636                 xiovec->iov_len = beio->sg_segs[i].len;
 637         }
 638
 639         binuptime(&beio->ds_t0);
 640         mtx_lock(&be_lun->io_lock);
 641         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 642         mtx_unlock(&be_lun->io_lock);
 643
 644         if (beio->bio_cmd == BIO_READ) {
 645                 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 646
 647                 /*
 648                  * UFS pays attention to IO_DIRECT for reads.  If the
 649                  * DIRECTIO option is configured into the kernel, it calls
 650                  * ffs_rawread().  But that only works for single-segment
 651                  * uios with user space addresses.  In our case, with a
 652                  * kernel uio, it still reads into the buffer cache, but it
 653                  * will just try to release the buffer from the cache later
 654                  * on in ffs_read().
 655                  *
 656                  * ZFS does not pay attention to IO_DIRECT for reads.
 657                  *
 658                  * UFS does not pay attention to IO_SYNC for reads.
 659                  *
 660                  * ZFS pays attention to IO_SYNC (which translates into the
 661                  * Solaris define FRSYNC for zfs_read()) for reads.  It
 662                  * attempts to sync the file before reading.
 663                  */
 664                 error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred);
 665
 666                 VOP_UNLOCK(be_lun->vn, 0);
 667                 SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
 668         } else {
 669                 struct mount *mountpoint;
 670                 int lock_flags;
 671
 672                 (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 673
 674                 if (MNT_SHARED_WRITES(mountpoint)
 675                  || ((mountpoint == NULL)
 676                   && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 677                         lock_flags = LK_SHARED;
 678                 else
 679                         lock_flags = LK_EXCLUSIVE;
 680
 681                 vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 682
 683                 /*
 684                  * UFS pays attention to IO_DIRECT for writes.  The write
 685                  * is done asynchronously.  (Normally the write would just
 686                  * get put into cache.
 687                  *
 688                  * UFS pays attention to IO_SYNC for writes.  It will
 689                  * attempt to write the buffer out synchronously if that
 690                  * flag is set.
 691                  *
 692                  * ZFS does not pay attention to IO_DIRECT for writes.
 693                  *
 694                  * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC)
 695                  * for writes.  It will flush the transaction from the
 696                  * cache before returning.
 697                  */
 698                 error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred);
 699                 VOP_UNLOCK(be_lun->vn, 0);
 700
 701                 vn_finished_write(mountpoint);
 702                 SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
 703         }
 704
 705         mtx_lock(&be_lun->io_lock);
 706         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 707             beio->ds_tag_type, beio->ds_trans_type,
 708             /*now*/ NULL, /*then*/&beio->ds_t0);
 709         mtx_unlock(&be_lun->io_lock);
 710
 711         /*
 712          * If we got an error, set the sense data to "MEDIUM ERROR" and
 713          * return the I/O to the user.
 714          */
 715         if (error != 0) {
 716                 char path_str[32];
 717
 718                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
 719                 printf("%s%s command returned errno %d\n", path_str,
 720                        (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error);
 721                 if (error == ENOSPC || error == EDQUOT) {
 722                         ctl_set_space_alloc_fail(&io->scsiio);
 723                 } else
 724                         ctl_set_medium_error(&io->scsiio);
 725                 ctl_complete_beio(beio);
 726                 return;
 727         }
 728
 729         /*
 730          * If this is a write or a verify, we're all done.
 731          * If this is a read, we can now send the data to the user.
 732          */
 733         if ((beio->bio_cmd == BIO_WRITE) ||
 734             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 735                 ctl_set_success(&io->scsiio);
 736                 ctl_complete_beio(beio);
 737         } else {
 738                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 739                     beio->beio_cont == NULL)
 740                         ctl_set_success(&io->scsiio);
 741 #ifdef CTL_TIME_IO
 742                 getbintime(&io->io_hdr.dma_start_bt);
 743 #endif
 744                 ctl_datamove(io);
 745         }
 746 }
 747
 748 static void
 749 ctl_be_block_gls_file(struct ctl_be_block_lun *be_lun,
 750                         struct ctl_be_block_io *beio)
 751 {
 752         union ctl_io *io = beio->io;
 753         struct ctl_lba_len_flags *lbalen = ARGS(io);
 754         struct scsi_get_lba_status_data *data;
 755         off_t roff, off;
 756         int error, status;
 757
 758         DPRINTF("entered\n");
 759
 760         off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
 761         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 762         error = VOP_IOCTL(be_lun->vn, FIOSEEKHOLE, &off,
 763             0, curthread->td_ucred, curthread);
 764         if (error == 0 && off > roff)
 765                 status = 0;     /* mapped up to off */
 766         else {
 767                 error = VOP_IOCTL(be_lun->vn, FIOSEEKDATA, &off,
 768                     0, curthread->td_ucred, curthread);
 769                 if (error == 0 && off > roff)
 770                         status = 1;     /* deallocated up to off */
 771                 else {
 772                         status = 0;     /* unknown up to the end */
 773                         off = be_lun->size_bytes;
 774                 }
 775         }
 776         VOP_UNLOCK(be_lun->vn, 0);
 777
 778         data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
 779         scsi_u64to8b(lbalen->lba, data->descr[0].addr);
 780         scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
 781             lbalen->lba), data->descr[0].length);
 782         data->descr[0].status = status;
 783
 784         ctl_complete_beio(beio);
 785 }
 786
 787 static uint64_t
 788 ctl_be_block_getattr_file(struct ctl_be_block_lun *be_lun, const char *attrname)
 789 {
 790         struct vattr            vattr;
 791         struct statfs           statfs;
 792         uint64_t                val;
 793         int                     error;
 794
 795         val = UINT64_MAX;
 796         if (be_lun->vn == NULL)
 797                 return (val);
 798         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 799         if (strcmp(attrname, "blocksused") == 0) {
 800                 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
 801                 if (error == 0)
 802                         val = vattr.va_bytes / be_lun->cbe_lun.blocksize;
 803         }
 804         if (strcmp(attrname, "blocksavail") == 0 &&
 805             (be_lun->vn->v_iflag & VI_DOOMED) == 0) {
 806                 error = VFS_STATFS(be_lun->vn->v_mount, &statfs);
 807                 if (error == 0)
 808                         val = statfs.f_bavail * statfs.f_bsize /
 809                             be_lun->cbe_lun.blocksize;
 810         }
 811         VOP_UNLOCK(be_lun->vn, 0);
 812         return (val);
 813 }
 814
 815 static void
 816 ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
 817                            struct ctl_be_block_io *beio)
 818 {
 819         union ctl_io *io;
 820         struct cdevsw *csw;
 821         struct cdev *dev;
 822         struct uio xuio;
 823         struct iovec *xiovec;
 824         int error, flags, i, ref;
 825
 826         DPRINTF("entered\n");
 827
 828         io = beio->io;
 829         flags = 0;
 830         if (ARGS(io)->flags & CTL_LLF_DPO)
 831                 flags |= IO_DIRECT;
 832         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 833                 flags |= IO_SYNC;
 834
 835         bzero(&xuio, sizeof(xuio));
 836         if (beio->bio_cmd == BIO_READ) {
 837                 SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0);
 838                 xuio.uio_rw = UIO_READ;
 839         } else {
 840                 SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0);
 841                 xuio.uio_rw = UIO_WRITE;
 842         }
 843         xuio.uio_offset = beio->io_offset;
 844         xuio.uio_resid = beio->io_len;
 845         xuio.uio_segflg = UIO_SYSSPACE;
 846         xuio.uio_iov = beio->xiovecs;
 847         xuio.uio_iovcnt = beio->num_segs;
 848         xuio.uio_td = curthread;
 849
 850         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 851                 xiovec->iov_base = beio->sg_segs[i].addr;
 852                 xiovec->iov_len = beio->sg_segs[i].len;
 853         }
 854
 855         binuptime(&beio->ds_t0);
 856         mtx_lock(&be_lun->io_lock);
 857         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 858         mtx_unlock(&be_lun->io_lock);
 859
 860         csw = devvn_refthread(be_lun->vn, &dev, &ref);
 861         if (csw) {
 862                 if (beio->bio_cmd == BIO_READ)
 863                         error = csw->d_read(dev, &xuio, flags);
 864                 else
 865                         error = csw->d_write(dev, &xuio, flags);
 866                 dev_relthread(dev, ref);
 867         } else
 868                 error = ENXIO;
 869
 870         if (beio->bio_cmd == BIO_READ)
 871                 SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
 872         else
 873                 SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
 874
 875         mtx_lock(&be_lun->io_lock);
 876         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 877             beio->ds_tag_type, beio->ds_trans_type,
 878             /*now*/ NULL, /*then*/&beio->ds_t0);
 879         mtx_unlock(&be_lun->io_lock);
 880
 881         /*
 882          * If we got an error, set the sense data to "MEDIUM ERROR" and
 883          * return the I/O to the user.
 884          */
 885         if (error != 0) {
 886                 if (error == ENOSPC || error == EDQUOT) {
 887                         ctl_set_space_alloc_fail(&io->scsiio);
 888                 } else
 889                         ctl_set_medium_error(&io->scsiio);
 890                 ctl_complete_beio(beio);
 891                 return;
 892         }
 893
 894         /*
 895          * If this is a write or a verify, we're all done.
 896          * If this is a read, we can now send the data to the user.
 897          */
 898         if ((beio->bio_cmd == BIO_WRITE) ||
 899             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 900                 ctl_set_success(&io->scsiio);
 901                 ctl_complete_beio(beio);
 902         } else {
 903                 if ((ARGS(io)->flags & CTL_LLF_READ) &&
 904                     beio->beio_cont == NULL)
 905                         ctl_set_success(&io->scsiio);
 906 #ifdef CTL_TIME_IO
 907                 getbintime(&io->io_hdr.dma_start_bt);
 908 #endif
 909                 ctl_datamove(io);
 910         }
 911 }
 912
 913 static void
 914 ctl_be_block_gls_zvol(struct ctl_be_block_lun *be_lun,
 915                         struct ctl_be_block_io *beio)
 916 {
 917         union ctl_io *io = beio->io;
 918         struct cdevsw *csw;
 919         struct cdev *dev;
 920         struct ctl_lba_len_flags *lbalen = ARGS(io);
 921         struct scsi_get_lba_status_data *data;
 922         off_t roff, off;
 923         int error, ref, status;
 924
 925         DPRINTF("entered\n");
 926
 927         csw = devvn_refthread(be_lun->vn, &dev, &ref);
 928         if (csw == NULL) {
 929                 status = 0;     /* unknown up to the end */
 930                 off = be_lun->size_bytes;
 931                 goto done;
 932         }
 933         off = roff = ((off_t)lbalen->lba) * be_lun->cbe_lun.blocksize;
 934         error = csw->d_ioctl(dev, FIOSEEKHOLE, (caddr_t)&off, FREAD,
 935             curthread);
 936         if (error == 0 && off > roff)
 937                 status = 0;     /* mapped up to off */
 938         else {
 939                 error = csw->d_ioctl(dev, FIOSEEKDATA, (caddr_t)&off, FREAD,
 940                     curthread);
 941                 if (error == 0 && off > roff)
 942                         status = 1;     /* deallocated up to off */
 943                 else {
 944                         status = 0;     /* unknown up to the end */
 945                         off = be_lun->size_bytes;
 946                 }
 947         }
 948         dev_relthread(dev, ref);
 949
 950 done:
 951         data = (struct scsi_get_lba_status_data *)io->scsiio.kern_data_ptr;
 952         scsi_u64to8b(lbalen->lba, data->descr[0].addr);
 953         scsi_ulto4b(MIN(UINT32_MAX, off / be_lun->cbe_lun.blocksize -
 954             lbalen->lba), data->descr[0].length);
 955         data->descr[0].status = status;
 956
 957         ctl_complete_beio(beio);
 958 }
 959
 960 static void
 961 ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 962                        struct ctl_be_block_io *beio)
 963 {
 964         struct bio *bio;
 965         union ctl_io *io;
 966         struct cdevsw *csw;
 967         struct cdev *dev;
 968         int ref;
 969
 970         io = beio->io;
 971
 972         DPRINTF("entered\n");
 973
 974         /* This can't fail, it's a blocking allocation. */
 975         bio = g_alloc_bio();
 976
 977         bio->bio_cmd        = BIO_FLUSH;
 978         bio->bio_offset     = 0;
 979         bio->bio_data       = 0;
 980         bio->bio_done       = ctl_be_block_biodone;
 981         bio->bio_caller1    = beio;
 982         bio->bio_pblkno     = 0;
 983
 984         /*
 985          * We don't need to acquire the LUN lock here, because we are only
 986          * sending one bio, and so there is no other context to synchronize
 987          * with.
 988          */
 989         beio->num_bios_sent = 1;
 990         beio->send_complete = 1;
 991
 992         binuptime(&beio->ds_t0);
 993         mtx_lock(&be_lun->io_lock);
 994         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
 995         mtx_unlock(&be_lun->io_lock);
 996
 997         csw = devvn_refthread(be_lun->vn, &dev, &ref);
 998         if (csw) {
 999                 bio->bio_dev = dev;
1000                 csw->d_strategy(bio);
1001                 dev_relthread(dev, ref);
1002         } else {
1003                 bio->bio_error = ENXIO;
1004                 ctl_be_block_biodone(bio);
1005         }
1006 }
1007
1008 static void
1009 ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun,
1010                        struct ctl_be_block_io *beio,
1011                        uint64_t off, uint64_t len, int last)
1012 {
1013         struct bio *bio;
1014         uint64_t maxlen;
1015         struct cdevsw *csw;
1016         struct cdev *dev;
1017         int ref;
1018
1019         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1020         maxlen = LONG_MAX - (LONG_MAX % be_lun->cbe_lun.blocksize);
1021         while (len > 0) {
1022                 bio = g_alloc_bio();
1023                 bio->bio_cmd        = BIO_DELETE;
1024                 bio->bio_dev        = dev;
1025                 bio->bio_offset     = off;
1026                 bio->bio_length     = MIN(len, maxlen);
1027                 bio->bio_data       = 0;
1028                 bio->bio_done       = ctl_be_block_biodone;
1029                 bio->bio_caller1    = beio;
1030                 bio->bio_pblkno     = off / be_lun->cbe_lun.blocksize;
1031
1032                 off += bio->bio_length;
1033                 len -= bio->bio_length;
1034
1035                 mtx_lock(&be_lun->io_lock);
1036                 beio->num_bios_sent++;
1037                 if (last && len == 0)
1038                         beio->send_complete = 1;
1039                 mtx_unlock(&be_lun->io_lock);
1040
1041                 if (csw) {
1042                         csw->d_strategy(bio);
1043                 } else {
1044                         bio->bio_error = ENXIO;
1045                         ctl_be_block_biodone(bio);
1046                 }
1047         }
1048         if (csw)
1049                 dev_relthread(dev, ref);
1050 }
1051
1052 static void
1053 ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
1054                        struct ctl_be_block_io *beio)
1055 {
1056         union ctl_io *io;
1057         struct ctl_ptr_len_flags *ptrlen;
1058         struct scsi_unmap_desc *buf, *end;
1059         uint64_t len;
1060
1061         io = beio->io;
1062
1063         DPRINTF("entered\n");
1064
1065         binuptime(&beio->ds_t0);
1066         mtx_lock(&be_lun->io_lock);
1067         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1068         mtx_unlock(&be_lun->io_lock);
1069
1070         if (beio->io_offset == -1) {
1071                 beio->io_len = 0;
1072                 ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1073                 buf = (struct scsi_unmap_desc *)ptrlen->ptr;
1074                 end = buf + ptrlen->len / sizeof(*buf);
1075                 for (; buf < end; buf++) {
1076                         len = (uint64_t)scsi_4btoul(buf->length) *
1077                             be_lun->cbe_lun.blocksize;
1078                         beio->io_len += len;
1079                         ctl_be_block_unmap_dev_range(be_lun, beio,
1080                             scsi_8btou64(buf->lba) * be_lun->cbe_lun.blocksize,
1081                             len, (end - buf < 2) ? TRUE : FALSE);
1082                 }
1083         } else
1084                 ctl_be_block_unmap_dev_range(be_lun, beio,
1085                     beio->io_offset, beio->io_len, TRUE);
1086 }
1087
1088 static void
1089 ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
1090                           struct ctl_be_block_io *beio)
1091 {
1092         TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue);
1093         struct bio *bio;
1094         struct cdevsw *csw;
1095         struct cdev *dev;
1096         off_t cur_offset;
1097         int i, max_iosize, ref;
1098
1099         DPRINTF("entered\n");
1100         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1101
1102         /*
1103          * We have to limit our I/O size to the maximum supported by the
1104          * backend device.  Hopefully it is MAXPHYS.  If the driver doesn't
1105          * set it properly, use DFLTPHYS.
1106          */
1107         if (csw) {
1108                 max_iosize = dev->si_iosize_max;
1109                 if (max_iosize < PAGE_SIZE)
1110                         max_iosize = DFLTPHYS;
1111         } else
1112                 max_iosize = DFLTPHYS;
1113
1114         cur_offset = beio->io_offset;
1115         for (i = 0; i < beio->num_segs; i++) {
1116                 size_t cur_size;
1117                 uint8_t *cur_ptr;
1118
1119                 cur_size = beio->sg_segs[i].len;
1120                 cur_ptr = beio->sg_segs[i].addr;
1121
1122                 while (cur_size > 0) {
1123                         /* This can't fail, it's a blocking allocation. */
1124                         bio = g_alloc_bio();
1125
1126                         KASSERT(bio != NULL, ("g_alloc_bio() failed!\n"));
1127
1128                         bio->bio_cmd = beio->bio_cmd;
1129                         bio->bio_dev = dev;
1130                         bio->bio_caller1 = beio;
1131                         bio->bio_length = min(cur_size, max_iosize);
1132                         bio->bio_offset = cur_offset;
1133                         bio->bio_data = cur_ptr;
1134                         bio->bio_done = ctl_be_block_biodone;
1135                         bio->bio_pblkno = cur_offset / be_lun->cbe_lun.blocksize;
1136
1137                         cur_offset += bio->bio_length;
1138                         cur_ptr += bio->bio_length;
1139                         cur_size -= bio->bio_length;
1140
1141                         TAILQ_INSERT_TAIL(&queue, bio, bio_queue);
1142                         beio->num_bios_sent++;
1143                 }
1144         }
1145         binuptime(&beio->ds_t0);
1146         mtx_lock(&be_lun->io_lock);
1147         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1148         beio->send_complete = 1;
1149         mtx_unlock(&be_lun->io_lock);
1150
1151         /*
1152          * Fire off all allocated requests!
1153          */
1154         while ((bio = TAILQ_FIRST(&queue)) != NULL) {
1155                 TAILQ_REMOVE(&queue, bio, bio_queue);
1156                 if (csw)
1157                         csw->d_strategy(bio);
1158                 else {
1159                         bio->bio_error = ENXIO;
1160                         ctl_be_block_biodone(bio);
1161                 }
1162         }
1163         if (csw)
1164                 dev_relthread(dev, ref);
1165 }
1166
1167 static uint64_t
1168 ctl_be_block_getattr_dev(struct ctl_be_block_lun *be_lun, const char *attrname)
1169 {
1170         struct diocgattr_arg    arg;
1171         struct cdevsw *csw;
1172         struct cdev *dev;
1173         int error, ref;
1174
1175         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1176         if (csw == NULL)
1177                 return (UINT64_MAX);
1178         strlcpy(arg.name, attrname, sizeof(arg.name));
1179         arg.len = sizeof(arg.value.off);
1180         if (csw->d_ioctl) {
1181                 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
1182                     curthread);
1183         } else
1184                 error = ENODEV;
1185         dev_relthread(dev, ref);
1186         if (error != 0)
1187                 return (UINT64_MAX);
1188         return (arg.value.off);
1189 }
1190
1191 static void
1192 ctl_be_block_cw_dispatch_sync(struct ctl_be_block_lun *be_lun,
1193                             union ctl_io *io)
1194 {
1195         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1196         struct ctl_be_block_io *beio;
1197         struct ctl_lba_len_flags *lbalen;
1198
1199         DPRINTF("entered\n");
1200         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1201         lbalen = (struct ctl_lba_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1202
1203         beio->io_len = lbalen->len * cbe_lun->blocksize;
1204         beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1205         beio->io_arg = (lbalen->flags & SSC_IMMED) != 0;
1206         beio->bio_cmd = BIO_FLUSH;
1207         beio->ds_trans_type = DEVSTAT_NO_DATA;
1208         DPRINTF("SYNC\n");
1209         be_lun->lun_flush(be_lun, beio);
1210 }
1211
1212 static void
1213 ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio)
1214 {
1215         union ctl_io *io;
1216
1217         io = beio->io;
1218         ctl_free_beio(beio);
1219         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1220             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1221              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1222                 ctl_config_write_done(io);
1223                 return;
1224         }
1225
1226         ctl_be_block_config_write(io);
1227 }
1228
1229 static void
1230 ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,
1231                             union ctl_io *io)
1232 {
1233         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1234         struct ctl_be_block_io *beio;
1235         struct ctl_lba_len_flags *lbalen;
1236         uint64_t len_left, lba;
1237         uint32_t pb, pbo, adj;
1238         int i, seglen;
1239         uint8_t *buf, *end;
1240
1241         DPRINTF("entered\n");
1242
1243         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1244         lbalen = ARGS(beio->io);
1245
1246         if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) ||
1247             (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) {
1248                 ctl_free_beio(beio);
1249                 ctl_set_invalid_field(&io->scsiio,
1250                                       /*sks_valid*/ 1,
1251                                       /*command*/ 1,
1252                                       /*field*/ 1,
1253                                       /*bit_valid*/ 0,
1254                                       /*bit*/ 0);
1255                 ctl_config_write_done(io);
1256                 return;
1257         }
1258
1259         if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) {
1260                 beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1261                 beio->io_len = (uint64_t)lbalen->len * cbe_lun->blocksize;
1262                 beio->bio_cmd = BIO_DELETE;
1263                 beio->ds_trans_type = DEVSTAT_FREE;
1264
1265                 be_lun->unmap(be_lun, beio);
1266                 return;
1267         }
1268
1269         beio->bio_cmd = BIO_WRITE;
1270         beio->ds_trans_type = DEVSTAT_WRITE;
1271
1272         DPRINTF("WRITE SAME at LBA %jx len %u\n",
1273                (uintmax_t)lbalen->lba, lbalen->len);
1274
1275         pb = cbe_lun->blocksize << be_lun->cbe_lun.pblockexp;
1276         if (be_lun->cbe_lun.pblockoff > 0)
1277                 pbo = pb - cbe_lun->blocksize * be_lun->cbe_lun.pblockoff;
1278         else
1279                 pbo = 0;
1280         len_left = (uint64_t)lbalen->len * cbe_lun->blocksize;
1281         for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) {
1282
1283                 /*
1284                  * Setup the S/G entry for this chunk.
1285                  */
1286                 seglen = MIN(CTLBLK_MAX_SEG, len_left);
1287                 if (pb > cbe_lun->blocksize) {
1288                         adj = ((lbalen->lba + lba) * cbe_lun->blocksize +
1289                             seglen - pbo) % pb;
1290                         if (seglen > adj)
1291                                 seglen -= adj;
1292                         else
1293                                 seglen -= seglen % cbe_lun->blocksize;
1294                 } else
1295                         seglen -= seglen % cbe_lun->blocksize;
1296                 beio->sg_segs[i].len = seglen;
1297                 beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1298
1299                 DPRINTF("segment %d addr %p len %zd\n", i,
1300                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1301
1302                 beio->num_segs++;
1303                 len_left -= seglen;
1304
1305                 buf = beio->sg_segs[i].addr;
1306                 end = buf + seglen;
1307                 for (; buf < end; buf += cbe_lun->blocksize) {
1308                         memcpy(buf, io->scsiio.kern_data_ptr, cbe_lun->blocksize);
1309                         if (lbalen->flags & SWS_LBDATA)
1310                                 scsi_ulto4b(lbalen->lba + lba, buf);
1311                         lba++;
1312                 }
1313         }
1314
1315         beio->io_offset = lbalen->lba * cbe_lun->blocksize;
1316         beio->io_len = lba * cbe_lun->blocksize;
1317
1318         /* We can not do all in one run. Correct and schedule rerun. */
1319         if (len_left > 0) {
1320                 lbalen->lba += lba;
1321                 lbalen->len -= lba;
1322                 beio->beio_cont = ctl_be_block_cw_done_ws;
1323         }
1324
1325         be_lun->dispatch(be_lun, beio);
1326 }
1327
1328 static void
1329 ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun,
1330                             union ctl_io *io)
1331 {
1332         struct ctl_be_block_io *beio;
1333         struct ctl_ptr_len_flags *ptrlen;
1334
1335         DPRINTF("entered\n");
1336
1337         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1338         ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1339
1340         if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) {
1341                 ctl_free_beio(beio);
1342                 ctl_set_invalid_field(&io->scsiio,
1343                                       /*sks_valid*/ 0,
1344                                       /*command*/ 1,
1345                                       /*field*/ 0,
1346                                       /*bit_valid*/ 0,
1347                                       /*bit*/ 0);
1348                 ctl_config_write_done(io);
1349                 return;
1350         }
1351
1352         beio->io_len = 0;
1353         beio->io_offset = -1;
1354         beio->bio_cmd = BIO_DELETE;
1355         beio->ds_trans_type = DEVSTAT_FREE;
1356         DPRINTF("UNMAP\n");
1357         be_lun->unmap(be_lun, beio);
1358 }
1359
1360 static void
1361 ctl_be_block_cr_done(struct ctl_be_block_io *beio)
1362 {
1363         union ctl_io *io;
1364
1365         io = beio->io;
1366         ctl_free_beio(beio);
1367         ctl_config_read_done(io);
1368 }
1369
1370 static void
1371 ctl_be_block_cr_dispatch(struct ctl_be_block_lun *be_lun,
1372                          union ctl_io *io)
1373 {
1374         struct ctl_be_block_io *beio;
1375         struct ctl_be_block_softc *softc;
1376
1377         DPRINTF("entered\n");
1378
1379         softc = be_lun->softc;
1380         beio = ctl_alloc_beio(softc);
1381         beio->io = io;
1382         beio->lun = be_lun;
1383         beio->beio_cont = ctl_be_block_cr_done;
1384         PRIV(io)->ptr = (void *)beio;
1385
1386         switch (io->scsiio.cdb[0]) {
1387         case SERVICE_ACTION_IN:         /* GET LBA STATUS */
1388                 beio->bio_cmd = -1;
1389                 beio->ds_trans_type = DEVSTAT_NO_DATA;
1390                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1391                 beio->io_len = 0;
1392                 if (be_lun->get_lba_status)
1393                         be_lun->get_lba_status(be_lun, beio);
1394                 else
1395                         ctl_be_block_cr_done(beio);
1396                 break;
1397         default:
1398                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1399                 break;
1400         }
1401 }
1402
1403 static void
1404 ctl_be_block_cw_done(struct ctl_be_block_io *beio)
1405 {
1406         union ctl_io *io;
1407
1408         io = beio->io;
1409         ctl_free_beio(beio);
1410         ctl_config_write_done(io);
1411 }
1412
1413 static void
1414 ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
1415                          union ctl_io *io)
1416 {
1417         struct ctl_be_block_io *beio;
1418         struct ctl_be_block_softc *softc;
1419
1420         DPRINTF("entered\n");
1421
1422         softc = be_lun->softc;
1423         beio = ctl_alloc_beio(softc);
1424         beio->io = io;
1425         beio->lun = be_lun;
1426         beio->beio_cont = ctl_be_block_cw_done;
1427         switch (io->scsiio.tag_type) {
1428         case CTL_TAG_ORDERED:
1429                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1430                 break;
1431         case CTL_TAG_HEAD_OF_QUEUE:
1432                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1433                 break;
1434         case CTL_TAG_UNTAGGED:
1435         case CTL_TAG_SIMPLE:
1436         case CTL_TAG_ACA:
1437         default:
1438                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1439                 break;
1440         }
1441         PRIV(io)->ptr = (void *)beio;
1442
1443         switch (io->scsiio.cdb[0]) {
1444         case SYNCHRONIZE_CACHE:
1445         case SYNCHRONIZE_CACHE_16:
1446                 ctl_be_block_cw_dispatch_sync(be_lun, io);
1447                 break;
1448         case WRITE_SAME_10:
1449         case WRITE_SAME_16:
1450                 ctl_be_block_cw_dispatch_ws(be_lun, io);
1451                 break;
1452         case UNMAP:
1453                 ctl_be_block_cw_dispatch_unmap(be_lun, io);
1454                 break;
1455         default:
1456                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1457                 break;
1458         }
1459 }
1460
1461 SDT_PROBE_DEFINE1(cbb, kernel, read, start, "uint64_t");
1462 SDT_PROBE_DEFINE1(cbb, kernel, write, start, "uint64_t");
1463 SDT_PROBE_DEFINE1(cbb, kernel, read, alloc_done, "uint64_t");
1464 SDT_PROBE_DEFINE1(cbb, kernel, write, alloc_done, "uint64_t");
1465
1466 static void
1467 ctl_be_block_next(struct ctl_be_block_io *beio)
1468 {
1469         struct ctl_be_block_lun *be_lun;
1470         union ctl_io *io;
1471
1472         io = beio->io;
1473         be_lun = beio->lun;
1474         ctl_free_beio(beio);
1475         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1476             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1477              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1478                 ctl_data_submit_done(io);
1479                 return;
1480         }
1481
1482         io->io_hdr.status &= ~CTL_STATUS_MASK;
1483         io->io_hdr.status |= CTL_STATUS_NONE;
1484
1485         mtx_lock(&be_lun->queue_lock);
1486         /*
1487          * XXX KDM make sure that links is okay to use at this point.
1488          * Otherwise, we either need to add another field to ctl_io_hdr,
1489          * or deal with resource allocation here.
1490          */
1491         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1492         mtx_unlock(&be_lun->queue_lock);
1493
1494         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1495 }
1496
1497 static void
1498 ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
1499                            union ctl_io *io)
1500 {
1501         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1502         struct ctl_be_block_io *beio;
1503         struct ctl_be_block_softc *softc;
1504         struct ctl_lba_len_flags *lbalen;
1505         struct ctl_ptr_len_flags *bptrlen;
1506         uint64_t len_left, lbas;
1507         int i;
1508
1509         softc = be_lun->softc;
1510
1511         DPRINTF("entered\n");
1512
1513         lbalen = ARGS(io);
1514         if (lbalen->flags & CTL_LLF_WRITE) {
1515                 SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0);
1516         } else {
1517                 SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0);
1518         }
1519
1520         beio = ctl_alloc_beio(softc);
1521         beio->io = io;
1522         beio->lun = be_lun;
1523         bptrlen = PRIV(io);
1524         bptrlen->ptr = (void *)beio;
1525
1526         switch (io->scsiio.tag_type) {
1527         case CTL_TAG_ORDERED:
1528                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1529                 break;
1530         case CTL_TAG_HEAD_OF_QUEUE:
1531                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1532                 break;
1533         case CTL_TAG_UNTAGGED:
1534         case CTL_TAG_SIMPLE:
1535         case CTL_TAG_ACA:
1536         default:
1537                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1538                 break;
1539         }
1540
1541         if (lbalen->flags & CTL_LLF_WRITE) {
1542                 beio->bio_cmd = BIO_WRITE;
1543                 beio->ds_trans_type = DEVSTAT_WRITE;
1544         } else {
1545                 beio->bio_cmd = BIO_READ;
1546                 beio->ds_trans_type = DEVSTAT_READ;
1547         }
1548
1549         DPRINTF("%s at LBA %jx len %u @%ju\n",
1550                (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
1551                (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len);
1552         if (lbalen->flags & CTL_LLF_COMPARE)
1553                 lbas = CTLBLK_HALF_IO_SIZE;
1554         else
1555                 lbas = CTLBLK_MAX_IO_SIZE;
1556         lbas = MIN(lbalen->len - bptrlen->len, lbas / cbe_lun->blocksize);
1557         beio->io_offset = (lbalen->lba + bptrlen->len) * cbe_lun->blocksize;
1558         beio->io_len = lbas * cbe_lun->blocksize;
1559         bptrlen->len += lbas;
1560
1561         for (i = 0, len_left = beio->io_len; len_left > 0; i++) {
1562                 KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)",
1563                     i, CTLBLK_MAX_SEGS));
1564
1565                 /*
1566                  * Setup the S/G entry for this chunk.
1567                  */
1568                 beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left);
1569                 beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1570
1571                 DPRINTF("segment %d addr %p len %zd\n", i,
1572                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1573
1574                 /* Set up second segment for compare operation. */
1575                 if (lbalen->flags & CTL_LLF_COMPARE) {
1576                         beio->sg_segs[i + CTLBLK_HALF_SEGS].len =
1577                             beio->sg_segs[i].len;
1578                         beio->sg_segs[i + CTLBLK_HALF_SEGS].addr =
1579                             uma_zalloc(be_lun->lun_zone, M_WAITOK);
1580                 }
1581
1582                 beio->num_segs++;
1583                 len_left -= beio->sg_segs[i].len;
1584         }
1585         if (bptrlen->len < lbalen->len)
1586                 beio->beio_cont = ctl_be_block_next;
1587         io->scsiio.be_move_done = ctl_be_block_move_done;
1588         /* For compare we have separate S/G lists for read and datamove. */
1589         if (lbalen->flags & CTL_LLF_COMPARE)
1590                 io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS];
1591         else
1592                 io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
1593         io->scsiio.kern_data_len = beio->io_len;
1594         io->scsiio.kern_data_resid = 0;
1595         io->scsiio.kern_sg_entries = beio->num_segs;
1596         io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
1597
1598         /*
1599          * For the read case, we need to read the data into our buffers and
1600          * then we can send it back to the user.  For the write case, we
1601          * need to get the data from the user first.
1602          */
1603         if (beio->bio_cmd == BIO_READ) {
1604                 SDT_PROBE(cbb, kernel, read, alloc_done, 0, 0, 0, 0, 0);
1605                 be_lun->dispatch(be_lun, beio);
1606         } else {
1607                 SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0);
1608 #ifdef CTL_TIME_IO
1609                 getbintime(&io->io_hdr.dma_start_bt);
1610 #endif
1611                 ctl_datamove(io);
1612         }
1613 }
1614
1615 static void
1616 ctl_be_block_worker(void *context, int pending)
1617 {
1618         struct ctl_be_block_lun *be_lun = (struct ctl_be_block_lun *)context;
1619         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1620         union ctl_io *io;
1621         struct ctl_be_block_io *beio;
1622
1623         DPRINTF("entered\n");
1624         /*
1625          * Fetch and process I/Os from all queues.  If we detect LUN
1626          * CTL_LUN_FLAG_OFFLINE status here -- it is result of a race,
1627          * so make response maximally opaque to not confuse initiator.
1628          */
1629         for (;;) {
1630                 mtx_lock(&be_lun->queue_lock);
1631                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);
1632                 if (io != NULL) {
1633                         DPRINTF("datamove queue\n");
1634                         STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr,
1635                                       ctl_io_hdr, links);
1636                         mtx_unlock(&be_lun->queue_lock);
1637                         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1638                         if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
1639                                 ctl_set_busy(&io->scsiio);
1640                                 ctl_complete_beio(beio);
1641                                 return;
1642                         }
1643                         be_lun->dispatch(be_lun, beio);
1644                         continue;
1645                 }
1646                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);
1647                 if (io != NULL) {
1648                         DPRINTF("config write queue\n");
1649                         STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr,
1650                                       ctl_io_hdr, links);
1651                         mtx_unlock(&be_lun->queue_lock);
1652                         if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
1653                                 ctl_set_busy(&io->scsiio);
1654                                 ctl_config_write_done(io);
1655                                 return;
1656                         }
1657                         ctl_be_block_cw_dispatch(be_lun, io);
1658                         continue;
1659                 }
1660                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_read_queue);
1661                 if (io != NULL) {
1662                         DPRINTF("config read queue\n");
1663                         STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr,
1664                                       ctl_io_hdr, links);
1665                         mtx_unlock(&be_lun->queue_lock);
1666                         if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
1667                                 ctl_set_busy(&io->scsiio);
1668                                 ctl_config_read_done(io);
1669                                 return;
1670                         }
1671                         ctl_be_block_cr_dispatch(be_lun, io);
1672                         continue;
1673                 }
1674                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);
1675                 if (io != NULL) {
1676                         DPRINTF("input queue\n");
1677                         STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr,
1678                                       ctl_io_hdr, links);
1679                         mtx_unlock(&be_lun->queue_lock);
1680                         if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
1681                                 ctl_set_busy(&io->scsiio);
1682                                 ctl_data_submit_done(io);
1683                                 return;
1684                         }
1685                         ctl_be_block_dispatch(be_lun, io);
1686                         continue;
1687                 }
1688
1689                 /*
1690                  * If we get here, there is no work left in the queues, so
1691                  * just break out and let the task queue go to sleep.
1692                  */
1693                 mtx_unlock(&be_lun->queue_lock);
1694                 break;
1695         }
1696 }
1697
1698 /*
1699  * Entry point from CTL to the backend for I/O.  We queue everything to a
1700  * work thread, so this just puts the I/O on a queue and wakes up the
1701  * thread.
1702  */
1703 static int
1704 ctl_be_block_submit(union ctl_io *io)
1705 {
1706         struct ctl_be_block_lun *be_lun;
1707         struct ctl_be_lun *cbe_lun;
1708
1709         DPRINTF("entered\n");
1710
1711         cbe_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
1712                 CTL_PRIV_BACKEND_LUN].ptr;
1713         be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
1714
1715         /*
1716          * Make sure we only get SCSI I/O.
1717          */
1718         KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type "
1719                 "%#x) encountered", io->io_hdr.io_type));
1720
1721         PRIV(io)->len = 0;
1722
1723         mtx_lock(&be_lun->queue_lock);
1724         /*
1725          * XXX KDM make sure that links is okay to use at this point.
1726          * Otherwise, we either need to add another field to ctl_io_hdr,
1727          * or deal with resource allocation here.
1728          */
1729         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1730         mtx_unlock(&be_lun->queue_lock);
1731         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1732
1733         return (CTL_RETVAL_COMPLETE);
1734 }
1735
1736 static int
1737 ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
1738                         int flag, struct thread *td)
1739 {
1740         struct ctl_be_block_softc *softc;
1741         int error;
1742
1743         softc = &backend_block_softc;
1744
1745         error = 0;
1746
1747         switch (cmd) {
1748         case CTL_LUN_REQ: {
1749                 struct ctl_lun_req *lun_req;
1750
1751                 lun_req = (struct ctl_lun_req *)addr;
1752
1753                 switch (lun_req->reqtype) {
1754                 case CTL_LUNREQ_CREATE:
1755                         error = ctl_be_block_create(softc, lun_req);
1756                         break;
1757                 case CTL_LUNREQ_RM:
1758                         error = ctl_be_block_rm(softc, lun_req);
1759                         break;
1760                 case CTL_LUNREQ_MODIFY:
1761                         error = ctl_be_block_modify(softc, lun_req);
1762                         break;
1763                 default:
1764                         lun_req->status = CTL_LUN_ERROR;
1765                         snprintf(lun_req->error_str, sizeof(lun_req->error_str),
1766                                  "invalid LUN request type %d",
1767                                  lun_req->reqtype);
1768                         break;
1769                 }
1770                 break;
1771         }
1772         default:
1773                 error = ENOTTY;
1774                 break;
1775         }
1776
1777         return (error);
1778 }
1779
1780 static int
1781 ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1782 {
1783         struct ctl_be_lun *cbe_lun;
1784         struct ctl_be_block_filedata *file_data;
1785         struct ctl_lun_create_params *params;
1786         char                         *value;
1787         struct vattr                  vattr;
1788         off_t                         ps, pss, po, pos, us, uss, uo, uos;
1789         int                           error;
1790
1791         error = 0;
1792         cbe_lun = &be_lun->cbe_lun;
1793         file_data = &be_lun->backend.file;
1794         params = &be_lun->params;
1795
1796         be_lun->dev_type = CTL_BE_BLOCK_FILE;
1797         be_lun->dispatch = ctl_be_block_dispatch_file;
1798         be_lun->lun_flush = ctl_be_block_flush_file;
1799         be_lun->get_lba_status = ctl_be_block_gls_file;
1800         be_lun->getattr = ctl_be_block_getattr_file;
1801         be_lun->unmap = NULL;
1802         cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
1803
1804         error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
1805         if (error != 0) {
1806                 snprintf(req->error_str, sizeof(req->error_str),
1807                          "error calling VOP_GETATTR() for file %s",
1808                          be_lun->dev_path);
1809                 return (error);
1810         }
1811
1812         /*
1813          * Verify that we have the ability to upgrade to exclusive
1814          * access on this file so we can trap errors at open instead
1815          * of reporting them during first access.
1816          */
1817         if (VOP_ISLOCKED(be_lun->vn) != LK_EXCLUSIVE) {
1818                 vn_lock(be_lun->vn, LK_UPGRADE | LK_RETRY);
1819                 if (be_lun->vn->v_iflag & VI_DOOMED) {
1820                         error = EBADF;
1821                         snprintf(req->error_str, sizeof(req->error_str),
1822                                  "error locking file %s", be_lun->dev_path);
1823                         return (error);
1824                 }
1825         }
1826
1827         file_data->cred = crhold(curthread->td_ucred);
1828         if (params->lun_size_bytes != 0)
1829                 be_lun->size_bytes = params->lun_size_bytes;
1830         else
1831                 be_lun->size_bytes = vattr.va_size;
1832
1833         /*
1834          * For files we can use any logical block size.  Prefer 512 bytes
1835          * for compatibility reasons.  If file's vattr.va_blocksize
1836          * (preferred I/O block size) is bigger and multiple to chosen
1837          * logical block size -- report it as physical block size.
1838          */
1839         if (params->blocksize_bytes != 0)
1840                 cbe_lun->blocksize = params->blocksize_bytes;
1841         else
1842                 cbe_lun->blocksize = 512;
1843         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
1844         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
1845             0 : (be_lun->size_blocks - 1);
1846
1847         us = ps = vattr.va_blocksize;
1848         uo = po = 0;
1849
1850         value = ctl_get_opt(&cbe_lun->options, "pblocksize");
1851         if (value != NULL)
1852                 ctl_expand_number(value, &ps);
1853         value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
1854         if (value != NULL)
1855                 ctl_expand_number(value, &po);
1856         pss = ps / cbe_lun->blocksize;
1857         pos = po / cbe_lun->blocksize;
1858         if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
1859             ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
1860                 cbe_lun->pblockexp = fls(pss) - 1;
1861                 cbe_lun->pblockoff = (pss - pos) % pss;
1862         }
1863
1864         value = ctl_get_opt(&cbe_lun->options, "ublocksize");
1865         if (value != NULL)
1866                 ctl_expand_number(value, &us);
1867         value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
1868         if (value != NULL)
1869                 ctl_expand_number(value, &uo);
1870         uss = us / cbe_lun->blocksize;
1871         uos = uo / cbe_lun->blocksize;
1872         if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
1873             ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
1874                 cbe_lun->ublockexp = fls(uss) - 1;
1875                 cbe_lun->ublockoff = (uss - uos) % uss;
1876         }
1877
1878         /*
1879          * Sanity check.  The media size has to be at least one
1880          * sector long.
1881          */
1882         if (be_lun->size_bytes < cbe_lun->blocksize) {
1883                 error = EINVAL;
1884                 snprintf(req->error_str, sizeof(req->error_str),
1885                          "file %s size %ju < block size %u", be_lun->dev_path,
1886                          (uintmax_t)be_lun->size_bytes, cbe_lun->blocksize);
1887         }
1888
1889         cbe_lun->opttxferlen = CTLBLK_MAX_IO_SIZE / cbe_lun->blocksize;
1890         return (error);
1891 }
1892
1893 static int
1894 ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1895 {
1896         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
1897         struct ctl_lun_create_params *params;
1898         struct cdevsw                *csw;
1899         struct cdev                  *dev;
1900         char                         *value;
1901         int                           error, atomic, maxio, ref, unmap, tmp;
1902         off_t                         ps, pss, po, pos, us, uss, uo, uos, otmp;
1903
1904         params = &be_lun->params;
1905
1906         be_lun->dev_type = CTL_BE_BLOCK_DEV;
1907         csw = devvn_refthread(be_lun->vn, &dev, &ref);
1908         if (csw == NULL)
1909                 return (ENXIO);
1910         if (strcmp(csw->d_name, "zvol") == 0) {
1911                 be_lun->dispatch = ctl_be_block_dispatch_zvol;
1912                 be_lun->get_lba_status = ctl_be_block_gls_zvol;
1913                 atomic = maxio = CTLBLK_MAX_IO_SIZE;
1914         } else {
1915                 be_lun->dispatch = ctl_be_block_dispatch_dev;
1916                 be_lun->get_lba_status = NULL;
1917                 atomic = 0;
1918                 maxio = dev->si_iosize_max;
1919                 if (maxio <= 0)
1920                         maxio = DFLTPHYS;
1921                 if (maxio > CTLBLK_MAX_IO_SIZE)
1922                         maxio = CTLBLK_MAX_IO_SIZE;
1923         }
1924         be_lun->lun_flush = ctl_be_block_flush_dev;
1925         be_lun->getattr = ctl_be_block_getattr_dev;
1926         be_lun->unmap = ctl_be_block_unmap_dev;
1927
1928         if (!csw->d_ioctl) {
1929                 dev_relthread(dev, ref);
1930                 snprintf(req->error_str, sizeof(req->error_str),
1931                          "no d_ioctl for device %s!", be_lun->dev_path);
1932                 return (ENODEV);
1933         }
1934
1935         error = csw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&tmp, FREAD,
1936                                curthread);
1937         if (error) {
1938                 dev_relthread(dev, ref);
1939                 snprintf(req->error_str, sizeof(req->error_str),
1940                          "error %d returned for DIOCGSECTORSIZE ioctl "
1941                          "on %s!", error, be_lun->dev_path);
1942                 return (error);
1943         }
1944
1945         /*
1946          * If the user has asked for a blocksize that is greater than the
1947          * backing device's blocksize, we can do it only if the blocksize
1948          * the user is asking for is an even multiple of the underlying
1949          * device's blocksize.
1950          */
1951         if ((params->blocksize_bytes != 0) &&
1952             (params->blocksize_bytes >= tmp)) {
1953                 if (params->blocksize_bytes % tmp == 0) {
1954                         cbe_lun->blocksize = params->blocksize_bytes;
1955                 } else {
1956                         dev_relthread(dev, ref);
1957                         snprintf(req->error_str, sizeof(req->error_str),
1958                                  "requested blocksize %u is not an even "
1959                                  "multiple of backing device blocksize %u",
1960                                  params->blocksize_bytes, tmp);
1961                         return (EINVAL);
1962                 }
1963         } else if (params->blocksize_bytes != 0) {
1964                 dev_relthread(dev, ref);
1965                 snprintf(req->error_str, sizeof(req->error_str),
1966                          "requested blocksize %u < backing device "
1967                          "blocksize %u", params->blocksize_bytes, tmp);
1968                 return (EINVAL);
1969         } else
1970                 cbe_lun->blocksize = tmp;
1971
1972         error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD,
1973                              curthread);
1974         if (error) {
1975                 dev_relthread(dev, ref);
1976                 snprintf(req->error_str, sizeof(req->error_str),
1977                          "error %d returned for DIOCGMEDIASIZE "
1978                          " ioctl on %s!", error,
1979                          be_lun->dev_path);
1980                 return (error);
1981         }
1982
1983         if (params->lun_size_bytes != 0) {
1984                 if (params->lun_size_bytes > otmp) {
1985                         dev_relthread(dev, ref);
1986                         snprintf(req->error_str, sizeof(req->error_str),
1987                                  "requested LUN size %ju > backing device "
1988                                  "size %ju",
1989                                  (uintmax_t)params->lun_size_bytes,
1990                                  (uintmax_t)otmp);
1991                         return (EINVAL);
1992                 }
1993
1994                 be_lun->size_bytes = params->lun_size_bytes;
1995         } else
1996                 be_lun->size_bytes = otmp;
1997         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
1998         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
1999             0 : (be_lun->size_blocks - 1);
2000
2001         error = csw->d_ioctl(dev, DIOCGSTRIPESIZE, (caddr_t)&ps, FREAD,
2002             curthread);
2003         if (error)
2004                 ps = po = 0;
2005         else {
2006                 error = csw->d_ioctl(dev, DIOCGSTRIPEOFFSET, (caddr_t)&po,
2007                     FREAD, curthread);
2008                 if (error)
2009                         po = 0;
2010         }
2011         us = ps;
2012         uo = po;
2013
2014         value = ctl_get_opt(&cbe_lun->options, "pblocksize");
2015         if (value != NULL)
2016                 ctl_expand_number(value, &ps);
2017         value = ctl_get_opt(&cbe_lun->options, "pblockoffset");
2018         if (value != NULL)
2019                 ctl_expand_number(value, &po);
2020         pss = ps / cbe_lun->blocksize;
2021         pos = po / cbe_lun->blocksize;
2022         if ((pss > 0) && (pss * cbe_lun->blocksize == ps) && (pss >= pos) &&
2023             ((pss & (pss - 1)) == 0) && (pos * cbe_lun->blocksize == po)) {
2024                 cbe_lun->pblockexp = fls(pss) - 1;
2025                 cbe_lun->pblockoff = (pss - pos) % pss;
2026         }
2027
2028         value = ctl_get_opt(&cbe_lun->options, "ublocksize");
2029         if (value != NULL)
2030                 ctl_expand_number(value, &us);
2031         value = ctl_get_opt(&cbe_lun->options, "ublockoffset");
2032         if (value != NULL)
2033                 ctl_expand_number(value, &uo);
2034         uss = us / cbe_lun->blocksize;
2035         uos = uo / cbe_lun->blocksize;
2036         if ((uss > 0) && (uss * cbe_lun->blocksize == us) && (uss >= uos) &&
2037             ((uss & (uss - 1)) == 0) && (uos * cbe_lun->blocksize == uo)) {
2038                 cbe_lun->ublockexp = fls(uss) - 1;
2039                 cbe_lun->ublockoff = (uss - uos) % uss;
2040         }
2041
2042         cbe_lun->atomicblock = atomic / cbe_lun->blocksize;
2043         cbe_lun->opttxferlen = maxio / cbe_lun->blocksize;
2044
2045         if (be_lun->dispatch == ctl_be_block_dispatch_zvol) {
2046                 unmap = 1;
2047         } else {
2048                 struct diocgattr_arg    arg;
2049
2050                 strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name));
2051                 arg.len = sizeof(arg.value.i);
2052                 error = csw->d_ioctl(dev, DIOCGATTR, (caddr_t)&arg, FREAD,
2053                     curthread);
2054                 unmap = (error == 0) ? arg.value.i : 0;
2055         }
2056         value = ctl_get_opt(&cbe_lun->options, "unmap");
2057         if (value != NULL)
2058                 unmap = (strcmp(value, "on") == 0);
2059         if (unmap)
2060                 cbe_lun->flags |= CTL_LUN_FLAG_UNMAP;
2061         else
2062                 cbe_lun->flags &= ~CTL_LUN_FLAG_UNMAP;
2063
2064         dev_relthread(dev, ref);
2065         return (0);
2066 }
2067
2068 static int
2069 ctl_be_block_close(struct ctl_be_block_lun *be_lun)
2070 {
2071         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2072         int flags;
2073
2074         if (be_lun->vn) {
2075                 flags = FREAD;
2076                 if ((cbe_lun->flags & CTL_LUN_FLAG_READONLY) == 0)
2077                         flags |= FWRITE;
2078                 (void)vn_close(be_lun->vn, flags, NOCRED, curthread);
2079                 be_lun->vn = NULL;
2080
2081                 switch (be_lun->dev_type) {
2082                 case CTL_BE_BLOCK_DEV:
2083                         break;
2084                 case CTL_BE_BLOCK_FILE:
2085                         if (be_lun->backend.file.cred != NULL) {
2086                                 crfree(be_lun->backend.file.cred);
2087                                 be_lun->backend.file.cred = NULL;
2088                         }
2089                         break;
2090                 case CTL_BE_BLOCK_NONE:
2091                         break;
2092                 default:
2093                         panic("Unexpected backend type.");
2094                         break;
2095                 }
2096                 be_lun->dev_type = CTL_BE_BLOCK_NONE;
2097         }
2098         return (0);
2099 }
2100
2101 static int
2102 ctl_be_block_open(struct ctl_be_block_softc *softc,
2103                   struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
2104 {
2105         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2106         struct nameidata nd;
2107         char            *value;
2108         int              error, flags;
2109
2110         error = 0;
2111         if (rootvnode == NULL) {
2112                 snprintf(req->error_str, sizeof(req->error_str),
2113                          "Root filesystem is not mounted");
2114                 return (1);
2115         }
2116         pwd_ensure_dirs();
2117
2118         value = ctl_get_opt(&cbe_lun->options, "file");
2119         if (value == NULL) {
2120                 snprintf(req->error_str, sizeof(req->error_str),
2121                          "no file argument specified");
2122                 return (1);
2123         }
2124         free(be_lun->dev_path, M_CTLBLK);
2125         be_lun->dev_path = strdup(value, M_CTLBLK);
2126
2127         flags = FREAD;
2128         value = ctl_get_opt(&cbe_lun->options, "readonly");
2129         if (value == NULL || strcmp(value, "on") != 0)
2130                 flags |= FWRITE;
2131
2132 again:
2133         NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread);
2134         error = vn_open(&nd, &flags, 0, NULL);
2135         if ((error == EROFS || error == EACCES) && (flags & FWRITE)) {
2136                 flags &= ~FWRITE;
2137                 goto again;
2138         }
2139         if (error) {
2140                 /*
2141                  * This is the only reasonable guess we can make as far as
2142                  * path if the user doesn't give us a fully qualified path.
2143                  * If they want to specify a file, they need to specify the
2144                  * full path.
2145                  */
2146                 if (be_lun->dev_path[0] != '/') {
2147                         char *dev_name;
2148
2149                         asprintf(&dev_name, M_CTLBLK, "/dev/%s",
2150                                 be_lun->dev_path);
2151                         free(be_lun->dev_path, M_CTLBLK);
2152                         be_lun->dev_path = dev_name;
2153                         goto again;
2154                 }
2155                 snprintf(req->error_str, sizeof(req->error_str),
2156                     "error opening %s: %d", be_lun->dev_path, error);
2157                 return (error);
2158         }
2159         if (flags & FWRITE)
2160                 cbe_lun->flags &= ~CTL_LUN_FLAG_READONLY;
2161         else
2162                 cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
2163
2164         NDFREE(&nd, NDF_ONLY_PNBUF);
2165         be_lun->vn = nd.ni_vp;
2166
2167         /* We only support disks and files. */
2168         if (vn_isdisk(be_lun->vn, &error)) {
2169                 error = ctl_be_block_open_dev(be_lun, req);
2170         } else if (be_lun->vn->v_type == VREG) {
2171                 error = ctl_be_block_open_file(be_lun, req);
2172         } else {
2173                 error = EINVAL;
2174                 snprintf(req->error_str, sizeof(req->error_str),
2175                          "%s is not a disk or plain file", be_lun->dev_path);
2176         }
2177         VOP_UNLOCK(be_lun->vn, 0);
2178
2179         if (error != 0)
2180                 ctl_be_block_close(be_lun);
2181         cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
2182         if (be_lun->dispatch != ctl_be_block_dispatch_dev)
2183                 cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
2184         value = ctl_get_opt(&cbe_lun->options, "serseq");
2185         if (value != NULL && strcmp(value, "on") == 0)
2186                 cbe_lun->serseq = CTL_LUN_SERSEQ_ON;
2187         else if (value != NULL && strcmp(value, "read") == 0)
2188                 cbe_lun->serseq = CTL_LUN_SERSEQ_READ;
2189         else if (value != NULL && strcmp(value, "off") == 0)
2190                 cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
2191         return (0);
2192 }
2193
2194 static int
2195 ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2196 {
2197         struct ctl_be_lun *cbe_lun;
2198         struct ctl_be_block_lun *be_lun;
2199         struct ctl_lun_create_params *params;
2200         char num_thread_str[16];
2201         char tmpstr[32];
2202         char *value;
2203         int retval, num_threads;
2204         int tmp_num_threads;
2205
2206         params = &req->reqdata.create;
2207         retval = 0;
2208         req->status = CTL_LUN_OK;
2209
2210         be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);
2211         cbe_lun = &be_lun->cbe_lun;
2212         cbe_lun->be_lun = be_lun;
2213         be_lun->params = req->reqdata.create;
2214         be_lun->softc = softc;
2215         STAILQ_INIT(&be_lun->input_queue);
2216         STAILQ_INIT(&be_lun->config_read_queue);
2217         STAILQ_INIT(&be_lun->config_write_queue);
2218         STAILQ_INIT(&be_lun->datamove_queue);
2219         sprintf(be_lun->lunname, "cblk%d", softc->num_luns);
2220         mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF);
2221         mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF);
2222         ctl_init_opts(&cbe_lun->options,
2223             req->num_be_args, req->kern_be_args);
2224         be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG,
2225             NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
2226         if (be_lun->lun_zone == NULL) {
2227                 snprintf(req->error_str, sizeof(req->error_str),
2228                          "error allocating UMA zone");
2229                 goto bailout_error;
2230         }
2231
2232         if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
2233                 cbe_lun->lun_type = params->device_type;
2234         else
2235                 cbe_lun->lun_type = T_DIRECT;
2236         be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED;
2237         cbe_lun->flags = 0;
2238         value = ctl_get_opt(&cbe_lun->options, "ha_role");
2239         if (value != NULL) {
2240                 if (strcmp(value, "primary") == 0)
2241                         cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2242         } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
2243                 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2244
2245         if (cbe_lun->lun_type == T_DIRECT) {
2246                 be_lun->size_bytes = params->lun_size_bytes;
2247                 if (params->blocksize_bytes != 0)
2248                         cbe_lun->blocksize = params->blocksize_bytes;
2249                 else
2250                         cbe_lun->blocksize = 512;
2251                 be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
2252                 cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
2253                     0 : (be_lun->size_blocks - 1);
2254
2255                 if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
2256                     control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
2257                         retval = ctl_be_block_open(softc, be_lun, req);
2258                         if (retval != 0) {
2259                                 retval = 0;
2260                                 req->status = CTL_LUN_WARNING;
2261                         }
2262                 }
2263                 num_threads = cbb_num_threads;
2264         } else {
2265                 num_threads = 1;
2266         }
2267
2268         /*
2269          * XXX This searching loop might be refactored to be combined with
2270          * the loop above,
2271          */
2272         value = ctl_get_opt(&cbe_lun->options, "num_threads");
2273         if (value != NULL) {
2274                 tmp_num_threads = strtol(value, NULL, 0);
2275
2276                 /*
2277                  * We don't let the user specify less than one
2278                  * thread, but hope he's clueful enough not to
2279                  * specify 1000 threads.
2280                  */
2281                 if (tmp_num_threads < 1) {
2282                         snprintf(req->error_str, sizeof(req->error_str),
2283                                  "invalid number of threads %s",
2284                                  num_thread_str);
2285                         goto bailout_error;
2286                 }
2287                 num_threads = tmp_num_threads;
2288         }
2289
2290         if (be_lun->vn == NULL)
2291                 cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE;
2292         /* Tell the user the blocksize we ended up using */
2293         params->lun_size_bytes = be_lun->size_bytes;
2294         params->blocksize_bytes = cbe_lun->blocksize;
2295         if (params->flags & CTL_LUN_FLAG_ID_REQ) {
2296                 cbe_lun->req_lun_id = params->req_lun_id;
2297                 cbe_lun->flags |= CTL_LUN_FLAG_ID_REQ;
2298         } else
2299                 cbe_lun->req_lun_id = 0;
2300
2301         cbe_lun->lun_shutdown = ctl_be_block_lun_shutdown;
2302         cbe_lun->lun_config_status = ctl_be_block_lun_config_status;
2303         cbe_lun->be = &ctl_be_block_driver;
2304
2305         if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) {
2306                 snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%4d",
2307                          softc->num_luns);
2308                 strncpy((char *)cbe_lun->serial_num, tmpstr,
2309                         MIN(sizeof(cbe_lun->serial_num), sizeof(tmpstr)));
2310
2311                 /* Tell the user what we used for a serial number */
2312                 strncpy((char *)params->serial_num, tmpstr,
2313                         MIN(sizeof(params->serial_num), sizeof(tmpstr)));
2314         } else {
2315                 strncpy((char *)cbe_lun->serial_num, params->serial_num,
2316                         MIN(sizeof(cbe_lun->serial_num),
2317                         sizeof(params->serial_num)));
2318         }
2319         if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) {
2320                 snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%4d", softc->num_luns);
2321                 strncpy((char *)cbe_lun->device_id, tmpstr,
2322                         MIN(sizeof(cbe_lun->device_id), sizeof(tmpstr)));
2323
2324                 /* Tell the user what we used for a device ID */
2325                 strncpy((char *)params->device_id, tmpstr,
2326                         MIN(sizeof(params->device_id), sizeof(tmpstr)));
2327         } else {
2328                 strncpy((char *)cbe_lun->device_id, params->device_id,
2329                         MIN(sizeof(cbe_lun->device_id),
2330                             sizeof(params->device_id)));
2331         }
2332
2333         TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);
2334
2335         be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK,
2336             taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);
2337
2338         if (be_lun->io_taskqueue == NULL) {
2339                 snprintf(req->error_str, sizeof(req->error_str),
2340                          "unable to create taskqueue");
2341                 goto bailout_error;
2342         }
2343
2344         /*
2345          * Note that we start the same number of threads by default for
2346          * both the file case and the block device case.  For the file
2347          * case, we need multiple threads to allow concurrency, because the
2348          * vnode interface is designed to be a blocking interface.  For the
2349          * block device case, ZFS zvols at least will block the caller's
2350          * context in many instances, and so we need multiple threads to
2351          * overcome that problem.  Other block devices don't need as many
2352          * threads, but they shouldn't cause too many problems.
2353          *
2354          * If the user wants to just have a single thread for a block
2355          * device, he can specify that when the LUN is created, or change
2356          * the tunable/sysctl to alter the default number of threads.
2357          */
2358         retval = taskqueue_start_threads(&be_lun->io_taskqueue,
2359                                          /*num threads*/num_threads,
2360                                          /*priority*/PWAIT,
2361                                          /*thread name*/
2362                                          "%s taskq", be_lun->lunname);
2363
2364         if (retval != 0)
2365                 goto bailout_error;
2366
2367         be_lun->num_threads = num_threads;
2368
2369         mtx_lock(&softc->lock);
2370         softc->num_luns++;
2371         STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links);
2372
2373         mtx_unlock(&softc->lock);
2374
2375         retval = ctl_add_lun(&be_lun->cbe_lun);
2376         if (retval != 0) {
2377                 mtx_lock(&softc->lock);
2378                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2379                               links);
2380                 softc->num_luns--;
2381                 mtx_unlock(&softc->lock);
2382                 snprintf(req->error_str, sizeof(req->error_str),
2383                          "ctl_add_lun() returned error %d, see dmesg for "
2384                          "details", retval);
2385                 retval = 0;
2386                 goto bailout_error;
2387         }
2388
2389         mtx_lock(&softc->lock);
2390
2391         /*
2392          * Tell the config_status routine that we're waiting so it won't
2393          * clean up the LUN in the event of an error.
2394          */
2395         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2396
2397         while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {
2398                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2399                 if (retval == EINTR)
2400                         break;
2401         }
2402         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2403
2404         if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) {
2405                 snprintf(req->error_str, sizeof(req->error_str),
2406                          "LUN configuration error, see dmesg for details");
2407                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2408                               links);
2409                 softc->num_luns--;
2410                 mtx_unlock(&softc->lock);
2411                 goto bailout_error;
2412         } else {
2413                 params->req_lun_id = cbe_lun->lun_id;
2414         }
2415
2416         mtx_unlock(&softc->lock);
2417
2418         be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id,
2419                                                cbe_lun->blocksize,
2420                                                DEVSTAT_ALL_SUPPORTED,
2421                                                cbe_lun->lun_type
2422                                                | DEVSTAT_TYPE_IF_OTHER,
2423                                                DEVSTAT_PRIORITY_OTHER);
2424
2425         return (retval);
2426
2427 bailout_error:
2428         req->status = CTL_LUN_ERROR;
2429
2430         if (be_lun->io_taskqueue != NULL)
2431                 taskqueue_free(be_lun->io_taskqueue);
2432         ctl_be_block_close(be_lun);
2433         if (be_lun->dev_path != NULL)
2434                 free(be_lun->dev_path, M_CTLBLK);
2435         if (be_lun->lun_zone != NULL)
2436                 uma_zdestroy(be_lun->lun_zone);
2437         ctl_free_opts(&cbe_lun->options);
2438         mtx_destroy(&be_lun->queue_lock);
2439         mtx_destroy(&be_lun->io_lock);
2440         free(be_lun, M_CTLBLK);
2441
2442         return (retval);
2443 }
2444
2445 static int
2446 ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2447 {
2448         struct ctl_lun_rm_params *params;
2449         struct ctl_be_block_lun *be_lun;
2450         struct ctl_be_lun *cbe_lun;
2451         int retval;
2452
2453         params = &req->reqdata.rm;
2454
2455         mtx_lock(&softc->lock);
2456         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2457                 if (be_lun->cbe_lun.lun_id == params->lun_id)
2458                         break;
2459         }
2460         mtx_unlock(&softc->lock);
2461
2462         if (be_lun == NULL) {
2463                 snprintf(req->error_str, sizeof(req->error_str),
2464                          "LUN %u is not managed by the block backend",
2465                          params->lun_id);
2466                 goto bailout_error;
2467         }
2468         cbe_lun = &be_lun->cbe_lun;
2469
2470         retval = ctl_disable_lun(cbe_lun);
2471         if (retval != 0) {
2472                 snprintf(req->error_str, sizeof(req->error_str),
2473                          "error %d returned from ctl_disable_lun() for "
2474                          "LUN %d", retval, params->lun_id);
2475                 goto bailout_error;
2476         }
2477
2478         if (be_lun->vn != NULL) {
2479                 cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE;
2480                 ctl_lun_offline(cbe_lun);
2481                 taskqueue_drain_all(be_lun->io_taskqueue);
2482                 ctl_be_block_close(be_lun);
2483         }
2484
2485         retval = ctl_invalidate_lun(cbe_lun);
2486         if (retval != 0) {
2487                 snprintf(req->error_str, sizeof(req->error_str),
2488                          "error %d returned from ctl_invalidate_lun() for "
2489                          "LUN %d", retval, params->lun_id);
2490                 goto bailout_error;
2491         }
2492
2493         mtx_lock(&softc->lock);
2494         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2495         while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2496                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2497                 if (retval == EINTR)
2498                         break;
2499         }
2500         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2501
2502         if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2503                 snprintf(req->error_str, sizeof(req->error_str),
2504                          "interrupted waiting for LUN to be freed");
2505                 mtx_unlock(&softc->lock);
2506                 goto bailout_error;
2507         }
2508
2509         STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links);
2510
2511         softc->num_luns--;
2512         mtx_unlock(&softc->lock);
2513
2514         taskqueue_drain_all(be_lun->io_taskqueue);
2515         taskqueue_free(be_lun->io_taskqueue);
2516
2517         if (be_lun->disk_stats != NULL)
2518                 devstat_remove_entry(be_lun->disk_stats);
2519
2520         uma_zdestroy(be_lun->lun_zone);
2521
2522         ctl_free_opts(&cbe_lun->options);
2523         free(be_lun->dev_path, M_CTLBLK);
2524         mtx_destroy(&be_lun->queue_lock);
2525         mtx_destroy(&be_lun->io_lock);
2526         free(be_lun, M_CTLBLK);
2527
2528         req->status = CTL_LUN_OK;
2529
2530         return (0);
2531
2532 bailout_error:
2533
2534         req->status = CTL_LUN_ERROR;
2535
2536         return (0);
2537 }
2538
2539 static int
2540 ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
2541                          struct ctl_lun_req *req)
2542 {
2543         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2544         struct vattr vattr;
2545         int error;
2546         struct ctl_lun_create_params *params = &be_lun->params;
2547
2548         if (params->lun_size_bytes != 0) {
2549                 be_lun->size_bytes = params->lun_size_bytes;
2550         } else  {
2551                 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
2552                 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
2553                 VOP_UNLOCK(be_lun->vn, 0);
2554                 if (error != 0) {
2555                         snprintf(req->error_str, sizeof(req->error_str),
2556                                  "error calling VOP_GETATTR() for file %s",
2557                                  be_lun->dev_path);
2558                         return (error);
2559                 }
2560                 be_lun->size_bytes = vattr.va_size;
2561         }
2562         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
2563         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
2564             0 : (be_lun->size_blocks - 1);
2565         return (0);
2566 }
2567
2568 static int
2569 ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
2570                         struct ctl_lun_req *req)
2571 {
2572         struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
2573         struct ctl_lun_create_params *params = &be_lun->params;
2574         struct cdevsw *csw;
2575         struct cdev *dev;
2576         uint64_t size_bytes;
2577         int error, ref;
2578
2579         csw = devvn_refthread(be_lun->vn, &dev, &ref);
2580         if (csw == NULL)
2581                 return (ENXIO);
2582         if (csw->d_ioctl == NULL) {
2583                 dev_relthread(dev, ref);
2584                 snprintf(req->error_str, sizeof(req->error_str),
2585                          "no d_ioctl for device %s!", be_lun->dev_path);
2586                 return (ENODEV);
2587         }
2588
2589         error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&size_bytes, FREAD,
2590             curthread);
2591         dev_relthread(dev, ref);
2592         if (error) {
2593                 snprintf(req->error_str, sizeof(req->error_str),
2594                          "error %d returned for DIOCGMEDIASIZE ioctl "
2595                          "on %s!", error, be_lun->dev_path);
2596                 return (error);
2597         }
2598
2599         if (params->lun_size_bytes != 0) {
2600                 if (params->lun_size_bytes > size_bytes) {
2601                         snprintf(req->error_str, sizeof(req->error_str),
2602                                  "requested LUN size %ju > backing device "
2603                                  "size %ju",
2604                                  (uintmax_t)params->lun_size_bytes,
2605                                  (uintmax_t)size_bytes);
2606                         return (EINVAL);
2607                 }
2608                 be_lun->size_bytes = params->lun_size_bytes;
2609         } else {
2610                 be_lun->size_bytes = size_bytes;
2611         }
2612         be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
2613         cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
2614             0 : (be_lun->size_blocks - 1);
2615         return (0);
2616 }
2617
2618 static int
2619 ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2620 {
2621         struct ctl_lun_modify_params *params;
2622         struct ctl_be_block_lun *be_lun;
2623         struct ctl_be_lun *cbe_lun;
2624         char *value;
2625         uint64_t oldsize;
2626         int error, wasprim;
2627
2628         params = &req->reqdata.modify;
2629
2630         mtx_lock(&softc->lock);
2631         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2632                 if (be_lun->cbe_lun.lun_id == params->lun_id)
2633                         break;
2634         }
2635         mtx_unlock(&softc->lock);
2636
2637         if (be_lun == NULL) {
2638                 snprintf(req->error_str, sizeof(req->error_str),
2639                          "LUN %u is not managed by the block backend",
2640                          params->lun_id);
2641                 goto bailout_error;
2642         }
2643         cbe_lun = &be_lun->cbe_lun;
2644
2645         if (params->lun_size_bytes != 0)
2646                 be_lun->params.lun_size_bytes = params->lun_size_bytes;
2647         ctl_update_opts(&cbe_lun->options, req->num_be_args, req->kern_be_args);
2648
2649         wasprim = (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY);
2650         value = ctl_get_opt(&cbe_lun->options, "ha_role");
2651         if (value != NULL) {
2652                 if (strcmp(value, "primary") == 0)
2653                         cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2654                 else
2655                         cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
2656         } else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
2657                 cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
2658         else
2659                 cbe_lun->flags &= ~CTL_LUN_FLAG_PRIMARY;
2660         if (wasprim != (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)) {
2661                 if (cbe_lun->flags & CTL_LUN_FLAG_PRIMARY)
2662                         ctl_lun_primary(cbe_lun);
2663                 else
2664                         ctl_lun_secondary(cbe_lun);
2665         }
2666
2667         oldsize = be_lun->size_blocks;
2668         if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
2669             control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
2670                 if (be_lun->vn == NULL)
2671                         error = ctl_be_block_open(softc, be_lun, req);
2672                 else if (vn_isdisk(be_lun->vn, &error))
2673                         error = ctl_be_block_modify_dev(be_lun, req);
2674                 else if (be_lun->vn->v_type == VREG)
2675                         error = ctl_be_block_modify_file(be_lun, req);
2676                 else
2677                         error = EINVAL;
2678                 if ((cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) &&
2679                     be_lun->vn != NULL) {
2680                         cbe_lun->flags &= ~CTL_LUN_FLAG_OFFLINE;
2681                         ctl_lun_online(cbe_lun);
2682                 }
2683         } else {
2684                 if (be_lun->vn != NULL) {
2685                         cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE;
2686                         ctl_lun_offline(cbe_lun);
2687                         taskqueue_drain_all(be_lun->io_taskqueue);
2688                         error = ctl_be_block_close(be_lun);
2689                 } else
2690                         error = 0;
2691         }
2692         if (be_lun->size_blocks != oldsize)
2693                 ctl_lun_capacity_changed(cbe_lun);
2694
2695         /* Tell the user the exact size we ended up using */
2696         params->lun_size_bytes = be_lun->size_bytes;
2697
2698         req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK;
2699         return (0);
2700
2701 bailout_error:
2702         req->status = CTL_LUN_ERROR;
2703         return (0);
2704 }
2705
2706 static void
2707 ctl_be_block_lun_shutdown(void *be_lun)
2708 {
2709         struct ctl_be_block_lun *lun;
2710         struct ctl_be_block_softc *softc;
2711
2712         lun = (struct ctl_be_block_lun *)be_lun;
2713
2714         softc = lun->softc;
2715
2716         mtx_lock(&softc->lock);
2717         lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;
2718         if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2719                 wakeup(lun);
2720         mtx_unlock(&softc->lock);
2721
2722 }
2723
2724 static void
2725 ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status)
2726 {
2727         struct ctl_be_block_lun *lun;
2728         struct ctl_be_block_softc *softc;
2729
2730         lun = (struct ctl_be_block_lun *)be_lun;
2731         softc = lun->softc;
2732
2733         if (status == CTL_LUN_CONFIG_OK) {
2734                 mtx_lock(&softc->lock);
2735                 lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2736                 if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2737                         wakeup(lun);
2738                 mtx_unlock(&softc->lock);
2739
2740                 /*
2741                  * We successfully added the LUN, attempt to enable it.
2742                  */
2743                 if (ctl_enable_lun(&lun->cbe_lun) != 0) {
2744                         printf("%s: ctl_enable_lun() failed!\n", __func__);
2745                         if (ctl_invalidate_lun(&lun->cbe_lun) != 0) {
2746                                 printf("%s: ctl_invalidate_lun() failed!\n",
2747                                        __func__);
2748                         }
2749                 }
2750
2751                 return;
2752         }
2753
2754
2755         mtx_lock(&softc->lock);
2756         lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2757         lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR;
2758         wakeup(lun);
2759         mtx_unlock(&softc->lock);
2760 }
2761
2762
2763 static int
2764 ctl_be_block_config_write(union ctl_io *io)
2765 {
2766         struct ctl_be_block_lun *be_lun;
2767         struct ctl_be_lun *cbe_lun;
2768         int retval;
2769
2770         retval = 0;
2771
2772         DPRINTF("entered\n");
2773
2774         cbe_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
2775                 CTL_PRIV_BACKEND_LUN].ptr;
2776         be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
2777
2778         switch (io->scsiio.cdb[0]) {
2779         case SYNCHRONIZE_CACHE:
2780         case SYNCHRONIZE_CACHE_16:
2781         case WRITE_SAME_10:
2782         case WRITE_SAME_16:
2783         case UNMAP:
2784                 /*
2785                  * The upper level CTL code will filter out any CDBs with
2786                  * the immediate bit set and return the proper error.
2787                  *
2788                  * We don't really need to worry about what LBA range the
2789                  * user asked to be synced out.  When they issue a sync
2790                  * cache command, we'll sync out the whole thing.
2791                  */
2792                 mtx_lock(&be_lun->queue_lock);
2793                 STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,
2794                                    links);
2795                 mtx_unlock(&be_lun->queue_lock);
2796                 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
2797                 break;
2798         case START_STOP_UNIT: {
2799                 struct scsi_start_stop_unit *cdb;
2800
2801                 cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
2802
2803                 if (cdb->how & SSS_START)
2804                         retval = ctl_start_lun(cbe_lun);
2805                 else {
2806                         retval = ctl_stop_lun(cbe_lun);
2807                         /*
2808                          * XXX KDM Copan-specific offline behavior.
2809                          * Figure out a reasonable way to port this?
2810                          */
2811 #ifdef NEEDTOPORT
2812                         if ((retval == 0)
2813                          && (cdb->byte2 & SSS_ONOFFLINE))
2814                                 retval = ctl_lun_offline(cbe_lun);
2815 #endif
2816                 }
2817
2818                 /*
2819                  * In general, the above routines should not fail.  They
2820                  * just set state for the LUN.  So we've got something
2821                  * pretty wrong here if we can't start or stop the LUN.
2822                  */
2823                 if (retval != 0) {
2824                         ctl_set_internal_failure(&io->scsiio,
2825                                                  /*sks_valid*/ 1,
2826                                                  /*retry_count*/ 0xf051);
2827                         retval = CTL_RETVAL_COMPLETE;
2828                 } else {
2829                         ctl_set_success(&io->scsiio);
2830                 }
2831                 ctl_config_write_done(io);
2832                 break;
2833         }
2834         default:
2835                 ctl_set_invalid_opcode(&io->scsiio);
2836                 ctl_config_write_done(io);
2837                 retval = CTL_RETVAL_COMPLETE;
2838                 break;
2839         }
2840
2841         return (retval);
2842 }
2843
2844 static int
2845 ctl_be_block_config_read(union ctl_io *io)
2846 {
2847         struct ctl_be_block_lun *be_lun;
2848         struct ctl_be_lun *cbe_lun;
2849         int retval = 0;
2850
2851         DPRINTF("entered\n");
2852
2853         cbe_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
2854                 CTL_PRIV_BACKEND_LUN].ptr;
2855         be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
2856
2857         switch (io->scsiio.cdb[0]) {
2858         case SERVICE_ACTION_IN:
2859                 if (io->scsiio.cdb[1] == SGLS_SERVICE_ACTION) {
2860                         mtx_lock(&be_lun->queue_lock);
2861                         STAILQ_INSERT_TAIL(&be_lun->config_read_queue,
2862                             &io->io_hdr, links);
2863                         mtx_unlock(&be_lun->queue_lock);
2864                         taskqueue_enqueue(be_lun->io_taskqueue,
2865                             &be_lun->io_task);
2866                         retval = CTL_RETVAL_QUEUED;
2867                         break;
2868                 }
2869                 ctl_set_invalid_field(&io->scsiio,
2870                                       /*sks_valid*/ 1,
2871                                       /*command*/ 1,
2872                                       /*field*/ 1,
2873                                       /*bit_valid*/ 1,
2874                                       /*bit*/ 4);
2875                 ctl_config_read_done(io);
2876                 retval = CTL_RETVAL_COMPLETE;
2877                 break;
2878         default:
2879                 ctl_set_invalid_opcode(&io->scsiio);
2880                 ctl_config_read_done(io);
2881                 retval = CTL_RETVAL_COMPLETE;
2882                 break;
2883         }
2884
2885         return (retval);
2886 }
2887
2888 static int
2889 ctl_be_block_lun_info(void *be_lun, struct sbuf *sb)
2890 {
2891         struct ctl_be_block_lun *lun;
2892         int retval;
2893
2894         lun = (struct ctl_be_block_lun *)be_lun;
2895         retval = 0;
2896
2897         retval = sbuf_printf(sb, "\t<num_threads>");
2898
2899         if (retval != 0)
2900                 goto bailout;
2901
2902         retval = sbuf_printf(sb, "%d", lun->num_threads);
2903
2904         if (retval != 0)
2905                 goto bailout;
2906
2907         retval = sbuf_printf(sb, "</num_threads>\n");
2908
2909 bailout:
2910
2911         return (retval);
2912 }
2913
2914 static uint64_t
2915 ctl_be_block_lun_attr(void *be_lun, const char *attrname)
2916 {
2917         struct ctl_be_block_lun *lun = (struct ctl_be_block_lun *)be_lun;
2918
2919         if (lun->getattr == NULL)
2920                 return (UINT64_MAX);
2921         return (lun->getattr(lun, attrname));
2922 }
2923
2924 int
2925 ctl_be_block_init(void)
2926 {
2927         struct ctl_be_block_softc *softc;
2928         int retval;
2929
2930         softc = &backend_block_softc;
2931         retval = 0;
2932
2933         mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF);
2934         beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io),
2935             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
2936         STAILQ_INIT(&softc->lun_list);
2937
2938         return (retval);
2939 }