sys/cam/ctl/ctl_backend_block.c

   1 /*-
   2  * Copyright (c) 2003 Silicon Graphics International Corp.
   3  * Copyright (c) 2009-2011 Spectra Logic Corporation
   4  * Copyright (c) 2012 The FreeBSD Foundation
   5  * All rights reserved.
   6  *
   7  * Portions of this software were developed by Edward Tomasz Napierala
   8  * under sponsorship from the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions, and the following disclaimer,
  15  *    without modification.
  16  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  17  *    substantially similar to the "NO WARRANTY" disclaimer below
  18  *    ("Disclaimer") and any redistribution must be conditioned upon
  19  *    including a substantially similar Disclaimer requirement for further
  20  *    binary redistribution.
  21  *
  22  * NO WARRANTY
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  32  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  33  * POSSIBILITY OF SUCH DAMAGES.
  34  *
  35  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
  36  */
  37 /*
  38  * CAM Target Layer driver backend for block devices.
  39  *
  40  * Author: Ken Merry <ken@FreeBSD.org>
  41  */
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 #include <opt_kdtrace.h>
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/limits.h>
  55 #include <sys/lock.h>
  56 #include <sys/mutex.h>
  57 #include <sys/condvar.h>
  58 #include <sys/malloc.h>
  59 #include <sys/conf.h>
  60 #include <sys/ioccom.h>
  61 #include <sys/queue.h>
  62 #include <sys/sbuf.h>
  63 #include <sys/endian.h>
  64 #include <sys/uio.h>
  65 #include <sys/buf.h>
  66 #include <sys/taskqueue.h>
  67 #include <sys/vnode.h>
  68 #include <sys/namei.h>
  69 #include <sys/mount.h>
  70 #include <sys/disk.h>
  71 #include <sys/fcntl.h>
  72 #include <sys/filedesc.h>
  73 #include <sys/proc.h>
  74 #include <sys/pcpu.h>
  75 #include <sys/module.h>
  76 #include <sys/sdt.h>
  77 #include <sys/devicestat.h>
  78 #include <sys/sysctl.h>
  79
  80 #include <geom/geom.h>
  81
  82 #include <cam/cam.h>
  83 #include <cam/scsi/scsi_all.h>
  84 #include <cam/scsi/scsi_da.h>
  85 #include <cam/ctl/ctl_io.h>
  86 #include <cam/ctl/ctl.h>
  87 #include <cam/ctl/ctl_backend.h>
  88 #include <cam/ctl/ctl_frontend_internal.h>
  89 #include <cam/ctl/ctl_ioctl.h>
  90 #include <cam/ctl/ctl_scsi_all.h>
  91 #include <cam/ctl/ctl_error.h>
  92
  93 /*
  94  * The idea here is that we'll allocate enough S/G space to hold a 1MB
  95  * I/O.  If we get an I/O larger than that, we'll split it.
  96  */
  97 #define CTLBLK_HALF_IO_SIZE     (512 * 1024)
  98 #define CTLBLK_MAX_IO_SIZE      (CTLBLK_HALF_IO_SIZE * 2)
  99 #define CTLBLK_MAX_SEG          MAXPHYS
 100 #define CTLBLK_HALF_SEGS        MAX(CTLBLK_HALF_IO_SIZE / CTLBLK_MAX_SEG, 1)
 101 #define CTLBLK_MAX_SEGS         (CTLBLK_HALF_SEGS * 2)
 102
 103 #ifdef CTLBLK_DEBUG
 104 #define DPRINTF(fmt, args...) \
 105     printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args)
 106 #else
 107 #define DPRINTF(fmt, args...) do {} while(0)
 108 #endif
 109
 110 #define PRIV(io)        \
 111     ((struct ctl_ptr_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_BACKEND])
 112 #define ARGS(io)        \
 113     ((struct ctl_lba_len_flags *)&(io)->io_hdr.ctl_private[CTL_PRIV_LBA_LEN])
 114
 115 SDT_PROVIDER_DEFINE(cbb);
 116
 117 typedef enum {
 118         CTL_BE_BLOCK_LUN_UNCONFIGURED   = 0x01,
 119         CTL_BE_BLOCK_LUN_CONFIG_ERR     = 0x02,
 120         CTL_BE_BLOCK_LUN_WAITING        = 0x04,
 121         CTL_BE_BLOCK_LUN_MULTI_THREAD   = 0x08
 122 } ctl_be_block_lun_flags;
 123
 124 typedef enum {
 125         CTL_BE_BLOCK_NONE,
 126         CTL_BE_BLOCK_DEV,
 127         CTL_BE_BLOCK_FILE
 128 } ctl_be_block_type;
 129
 130 struct ctl_be_block_devdata {
 131         struct cdev *cdev;
 132         struct cdevsw *csw;
 133         int dev_ref;
 134 };
 135
 136 struct ctl_be_block_filedata {
 137         struct ucred *cred;
 138 };
 139
 140 union ctl_be_block_bedata {
 141         struct ctl_be_block_devdata dev;
 142         struct ctl_be_block_filedata file;
 143 };
 144
 145 struct ctl_be_block_io;
 146 struct ctl_be_block_lun;
 147
 148 typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
 149                                struct ctl_be_block_io *beio);
 150
 151 /*
 152  * Backend LUN structure.  There is a 1:1 mapping between a block device
 153  * and a backend block LUN, and between a backend block LUN and a CTL LUN.
 154  */
 155 struct ctl_be_block_lun {
 156         struct ctl_lun_create_params params;
 157         struct ctl_block_disk *disk;
 158         char lunname[32];
 159         char *dev_path;
 160         ctl_be_block_type dev_type;
 161         struct vnode *vn;
 162         union ctl_be_block_bedata backend;
 163         cbb_dispatch_t dispatch;
 164         cbb_dispatch_t lun_flush;
 165         cbb_dispatch_t unmap;
 166         uma_zone_t lun_zone;
 167         uint64_t size_blocks;
 168         uint64_t size_bytes;
 169         uint32_t blocksize;
 170         int blocksize_shift;
 171         uint16_t pblockexp;
 172         uint16_t pblockoff;
 173         struct ctl_be_block_softc *softc;
 174         struct devstat *disk_stats;
 175         ctl_be_block_lun_flags flags;
 176         STAILQ_ENTRY(ctl_be_block_lun) links;
 177         struct ctl_be_lun ctl_be_lun;
 178         struct taskqueue *io_taskqueue;
 179         struct task io_task;
 180         int num_threads;
 181         STAILQ_HEAD(, ctl_io_hdr) input_queue;
 182         STAILQ_HEAD(, ctl_io_hdr) config_write_queue;
 183         STAILQ_HEAD(, ctl_io_hdr) datamove_queue;
 184         struct mtx_padalign io_lock;
 185         struct mtx_padalign queue_lock;
 186 };
 187
 188 /*
 189  * Overall softc structure for the block backend module.
 190  */
 191 struct ctl_be_block_softc {
 192         struct mtx                       lock;
 193         int                              num_disks;
 194         STAILQ_HEAD(, ctl_block_disk)    disk_list;
 195         int                              num_luns;
 196         STAILQ_HEAD(, ctl_be_block_lun)  lun_list;
 197 };
 198
 199 static struct ctl_be_block_softc backend_block_softc;
 200
 201 /*
 202  * Per-I/O information.
 203  */
 204 struct ctl_be_block_io {
 205         union ctl_io                    *io;
 206         struct ctl_sg_entry             sg_segs[CTLBLK_MAX_SEGS];
 207         struct iovec                    xiovecs[CTLBLK_MAX_SEGS];
 208         int                             bio_cmd;
 209         int                             num_segs;
 210         int                             num_bios_sent;
 211         int                             num_bios_done;
 212         int                             send_complete;
 213         int                             num_errors;
 214         struct bintime                  ds_t0;
 215         devstat_tag_type                ds_tag_type;
 216         devstat_trans_flags             ds_trans_type;
 217         uint64_t                        io_len;
 218         uint64_t                        io_offset;
 219         struct ctl_be_block_softc       *softc;
 220         struct ctl_be_block_lun         *lun;
 221         void (*beio_cont)(struct ctl_be_block_io *beio); /* to continue processing */
 222 };
 223
 224 static int cbb_num_threads = 14;
 225 TUNABLE_INT("kern.cam.ctl.block.num_threads", &cbb_num_threads);
 226 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0,
 227             "CAM Target Layer Block Backend");
 228 SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RW,
 229            &cbb_num_threads, 0, "Number of threads per backing file");
 230
 231 static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
 232 static void ctl_free_beio(struct ctl_be_block_io *beio);
 233 static void ctl_complete_beio(struct ctl_be_block_io *beio);
 234 static int ctl_be_block_move_done(union ctl_io *io);
 235 static void ctl_be_block_biodone(struct bio *bio);
 236 static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 237                                     struct ctl_be_block_io *beio);
 238 static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 239                                        struct ctl_be_block_io *beio);
 240 static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 241                                    struct ctl_be_block_io *beio);
 242 static void ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
 243                                    struct ctl_be_block_io *beio);
 244 static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
 245                                       struct ctl_be_block_io *beio);
 246 static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
 247                                     union ctl_io *io);
 248 static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 249                                   union ctl_io *io);
 250 static void ctl_be_block_worker(void *context, int pending);
 251 static int ctl_be_block_submit(union ctl_io *io);
 252 static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
 253                                    int flag, struct thread *td);
 254 static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
 255                                   struct ctl_lun_req *req);
 256 static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
 257                                  struct ctl_lun_req *req);
 258 static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
 259 static int ctl_be_block_open(struct ctl_be_block_softc *softc,
 260                              struct ctl_be_block_lun *be_lun,
 261                              struct ctl_lun_req *req);
 262 static int ctl_be_block_create(struct ctl_be_block_softc *softc,
 263                                struct ctl_lun_req *req);
 264 static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
 265                            struct ctl_lun_req *req);
 266 static int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
 267                                   struct ctl_lun_req *req);
 268 static int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
 269                                  struct ctl_lun_req *req);
 270 static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
 271                            struct ctl_lun_req *req);
 272 static void ctl_be_block_lun_shutdown(void *be_lun);
 273 static void ctl_be_block_lun_config_status(void *be_lun,
 274                                            ctl_lun_config_status status);
 275 static int ctl_be_block_config_write(union ctl_io *io);
 276 static int ctl_be_block_config_read(union ctl_io *io);
 277 static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb);
 278 int ctl_be_block_init(void);
 279
 280 static struct ctl_backend_driver ctl_be_block_driver =
 281 {
 282         .name = "block",
 283         .flags = CTL_BE_FLAG_HAS_CONFIG,
 284         .init = ctl_be_block_init,
 285         .data_submit = ctl_be_block_submit,
 286         .data_move_done = ctl_be_block_move_done,
 287         .config_read = ctl_be_block_config_read,
 288         .config_write = ctl_be_block_config_write,
 289         .ioctl = ctl_be_block_ioctl,
 290         .lun_info = ctl_be_block_lun_info
 291 };
 292
 293 MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend");
 294 CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
 295
 296 static uma_zone_t beio_zone;
 297
 298 static struct ctl_be_block_io *
 299 ctl_alloc_beio(struct ctl_be_block_softc *softc)
 300 {
 301         struct ctl_be_block_io *beio;
 302
 303         beio = uma_zalloc(beio_zone, M_WAITOK | M_ZERO);
 304         beio->softc = softc;
 305         return (beio);
 306 }
 307
 308 static void
 309 ctl_free_beio(struct ctl_be_block_io *beio)
 310 {
 311         int duplicate_free;
 312         int i;
 313
 314         duplicate_free = 0;
 315
 316         for (i = 0; i < beio->num_segs; i++) {
 317                 if (beio->sg_segs[i].addr == NULL)
 318                         duplicate_free++;
 319
 320                 uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr);
 321                 beio->sg_segs[i].addr = NULL;
 322
 323                 /* For compare we had two equal S/G lists. */
 324                 if (ARGS(beio->io)->flags & CTL_LLF_COMPARE) {
 325                         uma_zfree(beio->lun->lun_zone,
 326                             beio->sg_segs[i + CTLBLK_HALF_SEGS].addr);
 327                         beio->sg_segs[i + CTLBLK_HALF_SEGS].addr = NULL;
 328                 }
 329         }
 330
 331         if (duplicate_free > 0) {
 332                 printf("%s: %d duplicate frees out of %d segments\n", __func__,
 333                        duplicate_free, beio->num_segs);
 334         }
 335
 336         uma_zfree(beio_zone, beio);
 337 }
 338
 339 static void
 340 ctl_complete_beio(struct ctl_be_block_io *beio)
 341 {
 342         union ctl_io *io = beio->io;
 343
 344         if (beio->beio_cont != NULL) {
 345                 beio->beio_cont(beio);
 346         } else {
 347                 ctl_free_beio(beio);
 348                 ctl_data_submit_done(io);
 349         }
 350 }
 351
 352 static int
 353 ctl_be_block_move_done(union ctl_io *io)
 354 {
 355         struct ctl_be_block_io *beio;
 356         struct ctl_be_block_lun *be_lun;
 357         struct ctl_lba_len_flags *lbalen;
 358 #ifdef CTL_TIME_IO
 359         struct bintime cur_bt;
 360 #endif
 361         int i;
 362
 363         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 364         be_lun = beio->lun;
 365
 366         DPRINTF("entered\n");
 367
 368 #ifdef CTL_TIME_IO
 369         getbintime(&cur_bt);
 370         bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
 371         bintime_add(&io->io_hdr.dma_bt, &cur_bt);
 372         io->io_hdr.num_dmas++;
 373 #endif
 374         io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 375
 376         /*
 377          * We set status at this point for read commands, and write
 378          * commands with errors.
 379          */
 380         if ((io->io_hdr.port_status == 0) &&
 381             ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0) &&
 382             ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
 383                 lbalen = ARGS(beio->io);
 384                 if (lbalen->flags & CTL_LLF_READ) {
 385                         ctl_set_success(&io->scsiio);
 386                 } else if (lbalen->flags & CTL_LLF_COMPARE) {
 387                         /* We have two data blocks ready for comparison. */
 388                         for (i = 0; i < beio->num_segs; i++) {
 389                                 if (memcmp(beio->sg_segs[i].addr,
 390                                     beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
 391                                     beio->sg_segs[i].len) != 0)
 392                                         break;
 393                         }
 394                         if (i < beio->num_segs)
 395                                 ctl_set_sense(&io->scsiio,
 396                                     /*current_error*/ 1,
 397                                     /*sense_key*/ SSD_KEY_MISCOMPARE,
 398                                     /*asc*/ 0x1D,
 399                                     /*ascq*/ 0x00,
 400                                     SSD_ELEM_NONE);
 401                         else
 402                                 ctl_set_success(&io->scsiio);
 403                 }
 404         }
 405         else if ((io->io_hdr.port_status != 0)
 406               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
 407               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
 408                 /*
 409                  * For hardware error sense keys, the sense key
 410                  * specific value is defined to be a retry count,
 411                  * but we use it to pass back an internal FETD
 412                  * error code.  XXX KDM  Hopefully the FETD is only
 413                  * using 16 bits for an error code, since that's
 414                  * all the space we have in the sks field.
 415                  */
 416                 ctl_set_internal_failure(&io->scsiio,
 417                                          /*sks_valid*/ 1,
 418                                          /*retry_count*/
 419                                          io->io_hdr.port_status);
 420         }
 421
 422         /*
 423          * If this is a read, or a write with errors, it is done.
 424          */
 425         if ((beio->bio_cmd == BIO_READ)
 426          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)
 427          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) {
 428                 ctl_complete_beio(beio);
 429                 return (0);
 430         }
 431
 432         /*
 433          * At this point, we have a write and the DMA completed
 434          * successfully.  We now have to queue it to the task queue to
 435          * execute the backend I/O.  That is because we do blocking
 436          * memory allocations, and in the file backing case, blocking I/O.
 437          * This move done routine is generally called in the SIM's
 438          * interrupt context, and therefore we cannot block.
 439          */
 440         mtx_lock(&be_lun->queue_lock);
 441         /*
 442          * XXX KDM make sure that links is okay to use at this point.
 443          * Otherwise, we either need to add another field to ctl_io_hdr,
 444          * or deal with resource allocation here.
 445          */
 446         STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
 447         mtx_unlock(&be_lun->queue_lock);
 448
 449         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
 450
 451         return (0);
 452 }
 453
 454 static void
 455 ctl_be_block_biodone(struct bio *bio)
 456 {
 457         struct ctl_be_block_io *beio;
 458         struct ctl_be_block_lun *be_lun;
 459         union ctl_io *io;
 460         int error;
 461
 462         beio = bio->bio_caller1;
 463         be_lun = beio->lun;
 464         io = beio->io;
 465
 466         DPRINTF("entered\n");
 467
 468         error = bio->bio_error;
 469         mtx_lock(&be_lun->io_lock);
 470         if (error != 0)
 471                 beio->num_errors++;
 472
 473         beio->num_bios_done++;
 474
 475         /*
 476          * XXX KDM will this cause WITNESS to complain?  Holding a lock
 477          * during the free might cause it to complain.
 478          */
 479         g_destroy_bio(bio);
 480
 481         /*
 482          * If the send complete bit isn't set, or we aren't the last I/O to
 483          * complete, then we're done.
 484          */
 485         if ((beio->send_complete == 0)
 486          || (beio->num_bios_done < beio->num_bios_sent)) {
 487                 mtx_unlock(&be_lun->io_lock);
 488                 return;
 489         }
 490
 491         /*
 492          * At this point, we've verified that we are the last I/O to
 493          * complete, so it's safe to drop the lock.
 494          */
 495         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 496             beio->ds_tag_type, beio->ds_trans_type,
 497             /*now*/ NULL, /*then*/&beio->ds_t0);
 498         mtx_unlock(&be_lun->io_lock);
 499
 500         /*
 501          * If there are any errors from the backing device, we fail the
 502          * entire I/O with a medium error.
 503          */
 504         if (beio->num_errors > 0) {
 505                 if (error == EOPNOTSUPP) {
 506                         ctl_set_invalid_opcode(&io->scsiio);
 507                 } else if (beio->bio_cmd == BIO_FLUSH) {
 508                         /* XXX KDM is there is a better error here? */
 509                         ctl_set_internal_failure(&io->scsiio,
 510                                                  /*sks_valid*/ 1,
 511                                                  /*retry_count*/ 0xbad2);
 512                 } else
 513                         ctl_set_medium_error(&io->scsiio);
 514                 ctl_complete_beio(beio);
 515                 return;
 516         }
 517
 518         /*
 519          * If this is a write, a flush, a delete or verify, we're all done.
 520          * If this is a read, we can now send the data to the user.
 521          */
 522         if ((beio->bio_cmd == BIO_WRITE)
 523          || (beio->bio_cmd == BIO_FLUSH)
 524          || (beio->bio_cmd == BIO_DELETE)
 525          || (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 526                 ctl_set_success(&io->scsiio);
 527                 ctl_complete_beio(beio);
 528         } else {
 529 #ifdef CTL_TIME_IO
 530                 getbintime(&io->io_hdr.dma_start_bt);
 531 #endif
 532                 ctl_datamove(io);
 533         }
 534 }
 535
 536 static void
 537 ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 538                         struct ctl_be_block_io *beio)
 539 {
 540         union ctl_io *io = beio->io;
 541         struct mount *mountpoint;
 542         int error, lock_flags;
 543
 544         DPRINTF("entered\n");
 545
 546         binuptime(&beio->ds_t0);
 547         mtx_lock(&be_lun->io_lock);
 548         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 549         mtx_unlock(&be_lun->io_lock);
 550
 551         (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 552
 553         if (MNT_SHARED_WRITES(mountpoint)
 554          || ((mountpoint == NULL)
 555           && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 556                 lock_flags = LK_SHARED;
 557         else
 558                 lock_flags = LK_EXCLUSIVE;
 559
 560         vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 561
 562         error = VOP_FSYNC(be_lun->vn, MNT_WAIT, curthread);
 563         VOP_UNLOCK(be_lun->vn, 0);
 564
 565         vn_finished_write(mountpoint);
 566
 567         mtx_lock(&be_lun->io_lock);
 568         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 569             beio->ds_tag_type, beio->ds_trans_type,
 570             /*now*/ NULL, /*then*/&beio->ds_t0);
 571         mtx_unlock(&be_lun->io_lock);
 572
 573         if (error == 0)
 574                 ctl_set_success(&io->scsiio);
 575         else {
 576                 /* XXX KDM is there is a better error here? */
 577                 ctl_set_internal_failure(&io->scsiio,
 578                                          /*sks_valid*/ 1,
 579                                          /*retry_count*/ 0xbad1);
 580         }
 581
 582         ctl_complete_beio(beio);
 583 }
 584
 585 SDT_PROBE_DEFINE1(cbb, kernel, read, file_start, "uint64_t");
 586 SDT_PROBE_DEFINE1(cbb, kernel, write, file_start, "uint64_t");
 587 SDT_PROBE_DEFINE1(cbb, kernel, read, file_done,"uint64_t");
 588 SDT_PROBE_DEFINE1(cbb, kernel, write, file_done, "uint64_t");
 589
 590 static void
 591 ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 592                            struct ctl_be_block_io *beio)
 593 {
 594         struct ctl_be_block_filedata *file_data;
 595         union ctl_io *io;
 596         struct uio xuio;
 597         struct iovec *xiovec;
 598         int flags;
 599         int error, i;
 600
 601         DPRINTF("entered\n");
 602
 603         file_data = &be_lun->backend.file;
 604         io = beio->io;
 605         flags = 0;
 606         if (ARGS(io)->flags & CTL_LLF_DPO)
 607                 flags |= IO_DIRECT;
 608         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 609                 flags |= IO_SYNC;
 610
 611         bzero(&xuio, sizeof(xuio));
 612         if (beio->bio_cmd == BIO_READ) {
 613                 SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0);
 614                 xuio.uio_rw = UIO_READ;
 615         } else {
 616                 SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0);
 617                 xuio.uio_rw = UIO_WRITE;
 618         }
 619         xuio.uio_offset = beio->io_offset;
 620         xuio.uio_resid = beio->io_len;
 621         xuio.uio_segflg = UIO_SYSSPACE;
 622         xuio.uio_iov = beio->xiovecs;
 623         xuio.uio_iovcnt = beio->num_segs;
 624         xuio.uio_td = curthread;
 625
 626         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 627                 xiovec->iov_base = beio->sg_segs[i].addr;
 628                 xiovec->iov_len = beio->sg_segs[i].len;
 629         }
 630
 631         binuptime(&beio->ds_t0);
 632         mtx_lock(&be_lun->io_lock);
 633         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 634         mtx_unlock(&be_lun->io_lock);
 635
 636         if (beio->bio_cmd == BIO_READ) {
 637                 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 638
 639                 /*
 640                  * UFS pays attention to IO_DIRECT for reads.  If the
 641                  * DIRECTIO option is configured into the kernel, it calls
 642                  * ffs_rawread().  But that only works for single-segment
 643                  * uios with user space addresses.  In our case, with a
 644                  * kernel uio, it still reads into the buffer cache, but it
 645                  * will just try to release the buffer from the cache later
 646                  * on in ffs_read().
 647                  *
 648                  * ZFS does not pay attention to IO_DIRECT for reads.
 649                  *
 650                  * UFS does not pay attention to IO_SYNC for reads.
 651                  *
 652                  * ZFS pays attention to IO_SYNC (which translates into the
 653                  * Solaris define FRSYNC for zfs_read()) for reads.  It
 654                  * attempts to sync the file before reading.
 655                  *
 656                  * So, to attempt to provide some barrier semantics in the
 657                  * BIO_ORDERED case, set both IO_DIRECT and IO_SYNC.
 658                  */
 659                 error = VOP_READ(be_lun->vn, &xuio, flags, file_data->cred);
 660
 661                 VOP_UNLOCK(be_lun->vn, 0);
 662                 SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
 663         } else {
 664                 struct mount *mountpoint;
 665                 int lock_flags;
 666
 667                 (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 668
 669                 if (MNT_SHARED_WRITES(mountpoint)
 670                  || ((mountpoint == NULL)
 671                   && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 672                         lock_flags = LK_SHARED;
 673                 else
 674                         lock_flags = LK_EXCLUSIVE;
 675
 676                 vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 677
 678                 /*
 679                  * UFS pays attention to IO_DIRECT for writes.  The write
 680                  * is done asynchronously.  (Normally the write would just
 681                  * get put into cache.
 682                  *
 683                  * UFS pays attention to IO_SYNC for writes.  It will
 684                  * attempt to write the buffer out synchronously if that
 685                  * flag is set.
 686                  *
 687                  * ZFS does not pay attention to IO_DIRECT for writes.
 688                  *
 689                  * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC)
 690                  * for writes.  It will flush the transaction from the
 691                  * cache before returning.
 692                  *
 693                  * So if we've got the BIO_ORDERED flag set, we want
 694                  * IO_SYNC in either the UFS or ZFS case.
 695                  */
 696                 error = VOP_WRITE(be_lun->vn, &xuio, flags, file_data->cred);
 697                 VOP_UNLOCK(be_lun->vn, 0);
 698
 699                 vn_finished_write(mountpoint);
 700                 SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
 701         }
 702
 703         mtx_lock(&be_lun->io_lock);
 704         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 705             beio->ds_tag_type, beio->ds_trans_type,
 706             /*now*/ NULL, /*then*/&beio->ds_t0);
 707         mtx_unlock(&be_lun->io_lock);
 708
 709         /*
 710          * If we got an error, set the sense data to "MEDIUM ERROR" and
 711          * return the I/O to the user.
 712          */
 713         if (error != 0) {
 714                 char path_str[32];
 715
 716                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
 717                 /*
 718                  * XXX KDM ZFS returns ENOSPC when the underlying
 719                  * filesystem fills up.  What kind of SCSI error should we
 720                  * return for that?
 721                  */
 722                 printf("%s%s command returned errno %d\n", path_str,
 723                        (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error);
 724                 ctl_set_medium_error(&io->scsiio);
 725                 ctl_complete_beio(beio);
 726                 return;
 727         }
 728
 729         /*
 730          * If this is a write or a verify, we're all done.
 731          * If this is a read, we can now send the data to the user.
 732          */
 733         if ((beio->bio_cmd == BIO_WRITE) ||
 734             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 735                 ctl_set_success(&io->scsiio);
 736                 ctl_complete_beio(beio);
 737         } else {
 738 #ifdef CTL_TIME_IO
 739                 getbintime(&io->io_hdr.dma_start_bt);
 740 #endif
 741                 ctl_datamove(io);
 742         }
 743 }
 744
 745 static void
 746 ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
 747                            struct ctl_be_block_io *beio)
 748 {
 749         struct ctl_be_block_devdata *dev_data;
 750         union ctl_io *io;
 751         struct uio xuio;
 752         struct iovec *xiovec;
 753         int flags;
 754         int error, i;
 755
 756         DPRINTF("entered\n");
 757
 758         dev_data = &be_lun->backend.dev;
 759         io = beio->io;
 760         flags = 0;
 761         if (ARGS(io)->flags & CTL_LLF_DPO)
 762                 flags |= IO_DIRECT;
 763         if (beio->bio_cmd == BIO_WRITE && ARGS(io)->flags & CTL_LLF_FUA)
 764                 flags |= IO_SYNC;
 765
 766         bzero(&xuio, sizeof(xuio));
 767         if (beio->bio_cmd == BIO_READ) {
 768                 SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0);
 769                 xuio.uio_rw = UIO_READ;
 770         } else {
 771                 SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0);
 772                 xuio.uio_rw = UIO_WRITE;
 773         }
 774         xuio.uio_offset = beio->io_offset;
 775         xuio.uio_resid = beio->io_len;
 776         xuio.uio_segflg = UIO_SYSSPACE;
 777         xuio.uio_iov = beio->xiovecs;
 778         xuio.uio_iovcnt = beio->num_segs;
 779         xuio.uio_td = curthread;
 780
 781         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 782                 xiovec->iov_base = beio->sg_segs[i].addr;
 783                 xiovec->iov_len = beio->sg_segs[i].len;
 784         }
 785
 786         binuptime(&beio->ds_t0);
 787         mtx_lock(&be_lun->io_lock);
 788         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 789         mtx_unlock(&be_lun->io_lock);
 790
 791         if (beio->bio_cmd == BIO_READ) {
 792                 error = (*dev_data->csw->d_read)(dev_data->cdev, &xuio, flags);
 793                 SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
 794         } else {
 795                 error = (*dev_data->csw->d_write)(dev_data->cdev, &xuio, flags);
 796                 SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
 797         }
 798
 799         mtx_lock(&be_lun->io_lock);
 800         devstat_end_transaction(beio->lun->disk_stats, beio->io_len,
 801             beio->ds_tag_type, beio->ds_trans_type,
 802             /*now*/ NULL, /*then*/&beio->ds_t0);
 803         mtx_unlock(&be_lun->io_lock);
 804
 805         /*
 806          * If we got an error, set the sense data to "MEDIUM ERROR" and
 807          * return the I/O to the user.
 808          */
 809         if (error != 0) {
 810                 ctl_set_medium_error(&io->scsiio);
 811                 ctl_complete_beio(beio);
 812                 return;
 813         }
 814
 815         /*
 816          * If this is a write or a verify, we're all done.
 817          * If this is a read, we can now send the data to the user.
 818          */
 819         if ((beio->bio_cmd == BIO_WRITE) ||
 820             (ARGS(io)->flags & CTL_LLF_VERIFY)) {
 821                 ctl_set_success(&io->scsiio);
 822                 ctl_complete_beio(beio);
 823         } else {
 824 #ifdef CTL_TIME_IO
 825                 getbintime(&io->io_hdr.dma_start_bt);
 826 #endif
 827                 ctl_datamove(io);
 828         }
 829 }
 830
 831 static void
 832 ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 833                        struct ctl_be_block_io *beio)
 834 {
 835         struct bio *bio;
 836         union ctl_io *io;
 837         struct ctl_be_block_devdata *dev_data;
 838
 839         dev_data = &be_lun->backend.dev;
 840         io = beio->io;
 841
 842         DPRINTF("entered\n");
 843
 844         /* This can't fail, it's a blocking allocation. */
 845         bio = g_alloc_bio();
 846
 847         bio->bio_cmd        = BIO_FLUSH;
 848         bio->bio_flags     |= BIO_ORDERED;
 849         bio->bio_dev        = dev_data->cdev;
 850         bio->bio_offset     = 0;
 851         bio->bio_data       = 0;
 852         bio->bio_done       = ctl_be_block_biodone;
 853         bio->bio_caller1    = beio;
 854         bio->bio_pblkno     = 0;
 855
 856         /*
 857          * We don't need to acquire the LUN lock here, because we are only
 858          * sending one bio, and so there is no other context to synchronize
 859          * with.
 860          */
 861         beio->num_bios_sent = 1;
 862         beio->send_complete = 1;
 863
 864         binuptime(&beio->ds_t0);
 865         mtx_lock(&be_lun->io_lock);
 866         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
 867         mtx_unlock(&be_lun->io_lock);
 868
 869         (*dev_data->csw->d_strategy)(bio);
 870 }
 871
 872 static void
 873 ctl_be_block_unmap_dev_range(struct ctl_be_block_lun *be_lun,
 874                        struct ctl_be_block_io *beio,
 875                        uint64_t off, uint64_t len, int last)
 876 {
 877         struct bio *bio;
 878         struct ctl_be_block_devdata *dev_data;
 879         uint64_t maxlen;
 880
 881         dev_data = &be_lun->backend.dev;
 882         maxlen = LONG_MAX - (LONG_MAX % be_lun->blocksize);
 883         while (len > 0) {
 884                 bio = g_alloc_bio();
 885                 bio->bio_cmd        = BIO_DELETE;
 886                 bio->bio_dev        = dev_data->cdev;
 887                 bio->bio_offset     = off;
 888                 bio->bio_length     = MIN(len, maxlen);
 889                 bio->bio_data       = 0;
 890                 bio->bio_done       = ctl_be_block_biodone;
 891                 bio->bio_caller1    = beio;
 892                 bio->bio_pblkno     = off / be_lun->blocksize;
 893
 894                 off += bio->bio_length;
 895                 len -= bio->bio_length;
 896
 897                 mtx_lock(&be_lun->io_lock);
 898                 beio->num_bios_sent++;
 899                 if (last && len == 0)
 900                         beio->send_complete = 1;
 901                 mtx_unlock(&be_lun->io_lock);
 902
 903                 (*dev_data->csw->d_strategy)(bio);
 904         }
 905 }
 906
 907 static void
 908 ctl_be_block_unmap_dev(struct ctl_be_block_lun *be_lun,
 909                        struct ctl_be_block_io *beio)
 910 {
 911         union ctl_io *io;
 912         struct ctl_be_block_devdata *dev_data;
 913         struct ctl_ptr_len_flags *ptrlen;
 914         struct scsi_unmap_desc *buf, *end;
 915         uint64_t len;
 916
 917         dev_data = &be_lun->backend.dev;
 918         io = beio->io;
 919
 920         DPRINTF("entered\n");
 921
 922         binuptime(&beio->ds_t0);
 923         mtx_lock(&be_lun->io_lock);
 924         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
 925         mtx_unlock(&be_lun->io_lock);
 926
 927         if (beio->io_offset == -1) {
 928                 beio->io_len = 0;
 929                 ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
 930                 buf = (struct scsi_unmap_desc *)ptrlen->ptr;
 931                 end = buf + ptrlen->len / sizeof(*buf);
 932                 for (; buf < end; buf++) {
 933                         len = (uint64_t)scsi_4btoul(buf->length) *
 934                             be_lun->blocksize;
 935                         beio->io_len += len;
 936                         ctl_be_block_unmap_dev_range(be_lun, beio,
 937                             scsi_8btou64(buf->lba) * be_lun->blocksize, len,
 938                             (end - buf < 2) ? TRUE : FALSE);
 939                 }
 940         } else
 941                 ctl_be_block_unmap_dev_range(be_lun, beio,
 942                     beio->io_offset, beio->io_len, TRUE);
 943 }
 944
 945 static void
 946 ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
 947                           struct ctl_be_block_io *beio)
 948 {
 949         TAILQ_HEAD(, bio) queue = TAILQ_HEAD_INITIALIZER(queue);
 950         int i;
 951         struct bio *bio;
 952         struct ctl_be_block_devdata *dev_data;
 953         off_t cur_offset;
 954         int max_iosize;
 955
 956         DPRINTF("entered\n");
 957
 958         dev_data = &be_lun->backend.dev;
 959
 960         /*
 961          * We have to limit our I/O size to the maximum supported by the
 962          * backend device.  Hopefully it is MAXPHYS.  If the driver doesn't
 963          * set it properly, use DFLTPHYS.
 964          */
 965         max_iosize = dev_data->cdev->si_iosize_max;
 966         if (max_iosize < PAGE_SIZE)
 967                 max_iosize = DFLTPHYS;
 968
 969         cur_offset = beio->io_offset;
 970         for (i = 0; i < beio->num_segs; i++) {
 971                 size_t cur_size;
 972                 uint8_t *cur_ptr;
 973
 974                 cur_size = beio->sg_segs[i].len;
 975                 cur_ptr = beio->sg_segs[i].addr;
 976
 977                 while (cur_size > 0) {
 978                         /* This can't fail, it's a blocking allocation. */
 979                         bio = g_alloc_bio();
 980
 981                         KASSERT(bio != NULL, ("g_alloc_bio() failed!\n"));
 982
 983                         bio->bio_cmd = beio->bio_cmd;
 984                         bio->bio_dev = dev_data->cdev;
 985                         bio->bio_caller1 = beio;
 986                         bio->bio_length = min(cur_size, max_iosize);
 987                         bio->bio_offset = cur_offset;
 988                         bio->bio_data = cur_ptr;
 989                         bio->bio_done = ctl_be_block_biodone;
 990                         bio->bio_pblkno = cur_offset / be_lun->blocksize;
 991
 992                         cur_offset += bio->bio_length;
 993                         cur_ptr += bio->bio_length;
 994                         cur_size -= bio->bio_length;
 995
 996                         TAILQ_INSERT_TAIL(&queue, bio, bio_queue);
 997                         beio->num_bios_sent++;
 998                 }
 999         }
1000         binuptime(&beio->ds_t0);
1001         mtx_lock(&be_lun->io_lock);
1002         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
1003         beio->send_complete = 1;
1004         mtx_unlock(&be_lun->io_lock);
1005
1006         /*
1007          * Fire off all allocated requests!
1008          */
1009         while ((bio = TAILQ_FIRST(&queue)) != NULL) {
1010                 TAILQ_REMOVE(&queue, bio, bio_queue);
1011                 (*dev_data->csw->d_strategy)(bio);
1012         }
1013 }
1014
1015 static void
1016 ctl_be_block_cw_done_ws(struct ctl_be_block_io *beio)
1017 {
1018         union ctl_io *io;
1019
1020         io = beio->io;
1021         ctl_free_beio(beio);
1022         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1023             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1024              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1025                 ctl_config_write_done(io);
1026                 return;
1027         }
1028
1029         ctl_be_block_config_write(io);
1030 }
1031
1032 static void
1033 ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,
1034                             union ctl_io *io)
1035 {
1036         struct ctl_be_block_io *beio;
1037         struct ctl_be_block_softc *softc;
1038         struct ctl_lba_len_flags *lbalen;
1039         uint64_t len_left, lba;
1040         int i, seglen;
1041         uint8_t *buf, *end;
1042
1043         DPRINTF("entered\n");
1044
1045         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1046         softc = be_lun->softc;
1047         lbalen = ARGS(beio->io);
1048
1049         if (lbalen->flags & ~(SWS_LBDATA | SWS_UNMAP | SWS_ANCHOR | SWS_NDOB) ||
1050             (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR) && be_lun->unmap == NULL)) {
1051                 ctl_free_beio(beio);
1052                 ctl_set_invalid_field(&io->scsiio,
1053                                       /*sks_valid*/ 1,
1054                                       /*command*/ 1,
1055                                       /*field*/ 1,
1056                                       /*bit_valid*/ 0,
1057                                       /*bit*/ 0);
1058                 ctl_config_write_done(io);
1059                 return;
1060         }
1061
1062         switch (io->scsiio.tag_type) {
1063         case CTL_TAG_ORDERED:
1064                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1065                 break;
1066         case CTL_TAG_HEAD_OF_QUEUE:
1067                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1068                 break;
1069         case CTL_TAG_UNTAGGED:
1070         case CTL_TAG_SIMPLE:
1071         case CTL_TAG_ACA:
1072         default:
1073                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1074                 break;
1075         }
1076
1077         if (lbalen->flags & (SWS_UNMAP | SWS_ANCHOR)) {
1078                 beio->io_offset = lbalen->lba * be_lun->blocksize;
1079                 beio->io_len = (uint64_t)lbalen->len * be_lun->blocksize;
1080                 beio->bio_cmd = BIO_DELETE;
1081                 beio->ds_trans_type = DEVSTAT_FREE;
1082
1083                 be_lun->unmap(be_lun, beio);
1084                 return;
1085         }
1086
1087         beio->bio_cmd = BIO_WRITE;
1088         beio->ds_trans_type = DEVSTAT_WRITE;
1089
1090         DPRINTF("WRITE SAME at LBA %jx len %u\n",
1091                (uintmax_t)lbalen->lba, lbalen->len);
1092
1093         len_left = (uint64_t)lbalen->len * be_lun->blocksize;
1094         for (i = 0, lba = 0; i < CTLBLK_MAX_SEGS && len_left > 0; i++) {
1095
1096                 /*
1097                  * Setup the S/G entry for this chunk.
1098                  */
1099                 seglen = MIN(CTLBLK_MAX_SEG, len_left);
1100                 seglen -= seglen % be_lun->blocksize;
1101                 beio->sg_segs[i].len = seglen;
1102                 beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1103
1104                 DPRINTF("segment %d addr %p len %zd\n", i,
1105                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1106
1107                 beio->num_segs++;
1108                 len_left -= seglen;
1109
1110                 buf = beio->sg_segs[i].addr;
1111                 end = buf + seglen;
1112                 for (; buf < end; buf += be_lun->blocksize) {
1113                         memcpy(buf, io->scsiio.kern_data_ptr, be_lun->blocksize);
1114                         if (lbalen->flags & SWS_LBDATA)
1115                                 scsi_ulto4b(lbalen->lba + lba, buf);
1116                         lba++;
1117                 }
1118         }
1119
1120         beio->io_offset = lbalen->lba * be_lun->blocksize;
1121         beio->io_len = lba * be_lun->blocksize;
1122
1123         /* We can not do all in one run. Correct and schedule rerun. */
1124         if (len_left > 0) {
1125                 lbalen->lba += lba;
1126                 lbalen->len -= lba;
1127                 beio->beio_cont = ctl_be_block_cw_done_ws;
1128         }
1129
1130         be_lun->dispatch(be_lun, beio);
1131 }
1132
1133 static void
1134 ctl_be_block_cw_dispatch_unmap(struct ctl_be_block_lun *be_lun,
1135                             union ctl_io *io)
1136 {
1137         struct ctl_be_block_io *beio;
1138         struct ctl_be_block_softc *softc;
1139         struct ctl_ptr_len_flags *ptrlen;
1140
1141         DPRINTF("entered\n");
1142
1143         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1144         softc = be_lun->softc;
1145         ptrlen = (struct ctl_ptr_len_flags *)&io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
1146
1147         if ((ptrlen->flags & ~SU_ANCHOR) != 0 || be_lun->unmap == NULL) {
1148                 ctl_free_beio(beio);
1149                 ctl_set_invalid_field(&io->scsiio,
1150                                       /*sks_valid*/ 0,
1151                                       /*command*/ 1,
1152                                       /*field*/ 0,
1153                                       /*bit_valid*/ 0,
1154                                       /*bit*/ 0);
1155                 ctl_config_write_done(io);
1156                 return;
1157         }
1158
1159         switch (io->scsiio.tag_type) {
1160         case CTL_TAG_ORDERED:
1161                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1162                 break;
1163         case CTL_TAG_HEAD_OF_QUEUE:
1164                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1165                 break;
1166         case CTL_TAG_UNTAGGED:
1167         case CTL_TAG_SIMPLE:
1168         case CTL_TAG_ACA:
1169         default:
1170                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1171                 break;
1172         }
1173
1174         beio->io_len = 0;
1175         beio->io_offset = -1;
1176
1177         beio->bio_cmd = BIO_DELETE;
1178         beio->ds_trans_type = DEVSTAT_FREE;
1179
1180         DPRINTF("UNMAP\n");
1181
1182         be_lun->unmap(be_lun, beio);
1183 }
1184
1185 static void
1186 ctl_be_block_cw_done(struct ctl_be_block_io *beio)
1187 {
1188         union ctl_io *io;
1189
1190         io = beio->io;
1191         ctl_free_beio(beio);
1192         ctl_config_write_done(io);
1193 }
1194
1195 static void
1196 ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
1197                          union ctl_io *io)
1198 {
1199         struct ctl_be_block_io *beio;
1200         struct ctl_be_block_softc *softc;
1201
1202         DPRINTF("entered\n");
1203
1204         softc = be_lun->softc;
1205         beio = ctl_alloc_beio(softc);
1206         beio->io = io;
1207         beio->lun = be_lun;
1208         beio->beio_cont = ctl_be_block_cw_done;
1209         PRIV(io)->ptr = (void *)beio;
1210
1211         switch (io->scsiio.cdb[0]) {
1212         case SYNCHRONIZE_CACHE:
1213         case SYNCHRONIZE_CACHE_16:
1214                 beio->bio_cmd = BIO_FLUSH;
1215                 beio->ds_trans_type = DEVSTAT_NO_DATA;
1216                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1217                 beio->io_len = 0;
1218                 be_lun->lun_flush(be_lun, beio);
1219                 break;
1220         case WRITE_SAME_10:
1221         case WRITE_SAME_16:
1222                 ctl_be_block_cw_dispatch_ws(be_lun, io);
1223                 break;
1224         case UNMAP:
1225                 ctl_be_block_cw_dispatch_unmap(be_lun, io);
1226                 break;
1227         default:
1228                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
1229                 break;
1230         }
1231 }
1232
1233 SDT_PROBE_DEFINE1(cbb, kernel, read, start, "uint64_t");
1234 SDT_PROBE_DEFINE1(cbb, kernel, write, start, "uint64_t");
1235 SDT_PROBE_DEFINE1(cbb, kernel, read, alloc_done, "uint64_t");
1236 SDT_PROBE_DEFINE1(cbb, kernel, write, alloc_done, "uint64_t");
1237
1238 static void
1239 ctl_be_block_next(struct ctl_be_block_io *beio)
1240 {
1241         struct ctl_be_block_lun *be_lun;
1242         union ctl_io *io;
1243
1244         io = beio->io;
1245         be_lun = beio->lun;
1246         ctl_free_beio(beio);
1247         if ((io->io_hdr.flags & CTL_FLAG_ABORT) ||
1248             ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
1249              (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)) {
1250                 ctl_data_submit_done(io);
1251                 return;
1252         }
1253
1254         io->io_hdr.status &= ~CTL_STATUS_MASK;
1255         io->io_hdr.status |= CTL_STATUS_NONE;
1256
1257         mtx_lock(&be_lun->queue_lock);
1258         /*
1259          * XXX KDM make sure that links is okay to use at this point.
1260          * Otherwise, we either need to add another field to ctl_io_hdr,
1261          * or deal with resource allocation here.
1262          */
1263         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1264         mtx_unlock(&be_lun->queue_lock);
1265
1266         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1267 }
1268
1269 static void
1270 ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
1271                            union ctl_io *io)
1272 {
1273         struct ctl_be_block_io *beio;
1274         struct ctl_be_block_softc *softc;
1275         struct ctl_lba_len_flags *lbalen;
1276         struct ctl_ptr_len_flags *bptrlen;
1277         uint64_t len_left, lbas;
1278         int i;
1279
1280         softc = be_lun->softc;
1281
1282         DPRINTF("entered\n");
1283
1284         lbalen = ARGS(io);
1285         if (lbalen->flags & CTL_LLF_WRITE) {
1286                 SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0);
1287         } else {
1288                 SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0);
1289         }
1290
1291         beio = ctl_alloc_beio(softc);
1292         beio->io = io;
1293         beio->lun = be_lun;
1294         bptrlen = PRIV(io);
1295         bptrlen->ptr = (void *)beio;
1296
1297         switch (io->scsiio.tag_type) {
1298         case CTL_TAG_ORDERED:
1299                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1300                 break;
1301         case CTL_TAG_HEAD_OF_QUEUE:
1302                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1303                 break;
1304         case CTL_TAG_UNTAGGED:
1305         case CTL_TAG_SIMPLE:
1306         case CTL_TAG_ACA:
1307         default:
1308                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1309                 break;
1310         }
1311
1312         if (lbalen->flags & CTL_LLF_WRITE) {
1313                 beio->bio_cmd = BIO_WRITE;
1314                 beio->ds_trans_type = DEVSTAT_WRITE;
1315         } else {
1316                 beio->bio_cmd = BIO_READ;
1317                 beio->ds_trans_type = DEVSTAT_READ;
1318         }
1319
1320         DPRINTF("%s at LBA %jx len %u @%ju\n",
1321                (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
1322                (uintmax_t)lbalen->lba, lbalen->len, bptrlen->len);
1323         if (lbalen->flags & CTL_LLF_COMPARE)
1324                 lbas = CTLBLK_HALF_IO_SIZE;
1325         else
1326                 lbas = CTLBLK_MAX_IO_SIZE;
1327         lbas = MIN(lbalen->len - bptrlen->len, lbas / be_lun->blocksize);
1328         beio->io_offset = (lbalen->lba + bptrlen->len) * be_lun->blocksize;
1329         beio->io_len = lbas * be_lun->blocksize;
1330         bptrlen->len += lbas;
1331
1332         for (i = 0, len_left = beio->io_len; len_left > 0; i++) {
1333                 KASSERT(i < CTLBLK_MAX_SEGS, ("Too many segs (%d >= %d)",
1334                     i, CTLBLK_MAX_SEGS));
1335
1336                 /*
1337                  * Setup the S/G entry for this chunk.
1338                  */
1339                 beio->sg_segs[i].len = min(CTLBLK_MAX_SEG, len_left);
1340                 beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1341
1342                 DPRINTF("segment %d addr %p len %zd\n", i,
1343                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1344
1345                 /* Set up second segment for compare operation. */
1346                 if (lbalen->flags & CTL_LLF_COMPARE) {
1347                         beio->sg_segs[i + CTLBLK_HALF_SEGS].len =
1348                             beio->sg_segs[i].len;
1349                         beio->sg_segs[i + CTLBLK_HALF_SEGS].addr =
1350                             uma_zalloc(be_lun->lun_zone, M_WAITOK);
1351                 }
1352
1353                 beio->num_segs++;
1354                 len_left -= beio->sg_segs[i].len;
1355         }
1356         if (bptrlen->len < lbalen->len)
1357                 beio->beio_cont = ctl_be_block_next;
1358         io->scsiio.be_move_done = ctl_be_block_move_done;
1359         /* For compare we have separate S/G lists for read and datamove. */
1360         if (lbalen->flags & CTL_LLF_COMPARE)
1361                 io->scsiio.kern_data_ptr = (uint8_t *)&beio->sg_segs[CTLBLK_HALF_SEGS];
1362         else
1363                 io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
1364         io->scsiio.kern_data_len = beio->io_len;
1365         io->scsiio.kern_data_resid = 0;
1366         io->scsiio.kern_sg_entries = beio->num_segs;
1367         io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
1368
1369         /*
1370          * For the read case, we need to read the data into our buffers and
1371          * then we can send it back to the user.  For the write case, we
1372          * need to get the data from the user first.
1373          */
1374         if (beio->bio_cmd == BIO_READ) {
1375                 SDT_PROBE(cbb, kernel, read, alloc_done, 0, 0, 0, 0, 0);
1376                 be_lun->dispatch(be_lun, beio);
1377         } else {
1378                 SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0);
1379 #ifdef CTL_TIME_IO
1380                 getbintime(&io->io_hdr.dma_start_bt);
1381 #endif
1382                 ctl_datamove(io);
1383         }
1384 }
1385
1386 static void
1387 ctl_be_block_worker(void *context, int pending)
1388 {
1389         struct ctl_be_block_lun *be_lun;
1390         struct ctl_be_block_softc *softc;
1391         union ctl_io *io;
1392
1393         be_lun = (struct ctl_be_block_lun *)context;
1394         softc = be_lun->softc;
1395
1396         DPRINTF("entered\n");
1397
1398         mtx_lock(&be_lun->queue_lock);
1399         for (;;) {
1400                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);
1401                 if (io != NULL) {
1402                         struct ctl_be_block_io *beio;
1403
1404                         DPRINTF("datamove queue\n");
1405
1406                         STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr,
1407                                       ctl_io_hdr, links);
1408
1409                         mtx_unlock(&be_lun->queue_lock);
1410
1411                         beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
1412
1413                         be_lun->dispatch(be_lun, beio);
1414
1415                         mtx_lock(&be_lun->queue_lock);
1416                         continue;
1417                 }
1418                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);
1419                 if (io != NULL) {
1420
1421                         DPRINTF("config write queue\n");
1422
1423                         STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr,
1424                                       ctl_io_hdr, links);
1425
1426                         mtx_unlock(&be_lun->queue_lock);
1427
1428                         ctl_be_block_cw_dispatch(be_lun, io);
1429
1430                         mtx_lock(&be_lun->queue_lock);
1431                         continue;
1432                 }
1433                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);
1434                 if (io != NULL) {
1435                         DPRINTF("input queue\n");
1436
1437                         STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr,
1438                                       ctl_io_hdr, links);
1439                         mtx_unlock(&be_lun->queue_lock);
1440
1441                         /*
1442                          * We must drop the lock, since this routine and
1443                          * its children may sleep.
1444                          */
1445                         ctl_be_block_dispatch(be_lun, io);
1446
1447                         mtx_lock(&be_lun->queue_lock);
1448                         continue;
1449                 }
1450
1451                 /*
1452                  * If we get here, there is no work left in the queues, so
1453                  * just break out and let the task queue go to sleep.
1454                  */
1455                 break;
1456         }
1457         mtx_unlock(&be_lun->queue_lock);
1458 }
1459
1460 /*
1461  * Entry point from CTL to the backend for I/O.  We queue everything to a
1462  * work thread, so this just puts the I/O on a queue and wakes up the
1463  * thread.
1464  */
1465 static int
1466 ctl_be_block_submit(union ctl_io *io)
1467 {
1468         struct ctl_be_block_lun *be_lun;
1469         struct ctl_be_lun *ctl_be_lun;
1470
1471         DPRINTF("entered\n");
1472
1473         ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
1474                 CTL_PRIV_BACKEND_LUN].ptr;
1475         be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun;
1476
1477         /*
1478          * Make sure we only get SCSI I/O.
1479          */
1480         KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type "
1481                 "%#x) encountered", io->io_hdr.io_type));
1482
1483         PRIV(io)->len = 0;
1484
1485         mtx_lock(&be_lun->queue_lock);
1486         /*
1487          * XXX KDM make sure that links is okay to use at this point.
1488          * Otherwise, we either need to add another field to ctl_io_hdr,
1489          * or deal with resource allocation here.
1490          */
1491         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1492         mtx_unlock(&be_lun->queue_lock);
1493         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1494
1495         return (CTL_RETVAL_COMPLETE);
1496 }
1497
1498 static int
1499 ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
1500                         int flag, struct thread *td)
1501 {
1502         struct ctl_be_block_softc *softc;
1503         int error;
1504
1505         softc = &backend_block_softc;
1506
1507         error = 0;
1508
1509         switch (cmd) {
1510         case CTL_LUN_REQ: {
1511                 struct ctl_lun_req *lun_req;
1512
1513                 lun_req = (struct ctl_lun_req *)addr;
1514
1515                 switch (lun_req->reqtype) {
1516                 case CTL_LUNREQ_CREATE:
1517                         error = ctl_be_block_create(softc, lun_req);
1518                         break;
1519                 case CTL_LUNREQ_RM:
1520                         error = ctl_be_block_rm(softc, lun_req);
1521                         break;
1522                 case CTL_LUNREQ_MODIFY:
1523                         error = ctl_be_block_modify(softc, lun_req);
1524                         break;
1525                 default:
1526                         lun_req->status = CTL_LUN_ERROR;
1527                         snprintf(lun_req->error_str, sizeof(lun_req->error_str),
1528                                  "invalid LUN request type %d",
1529                                  lun_req->reqtype);
1530                         break;
1531                 }
1532                 break;
1533         }
1534         default:
1535                 error = ENOTTY;
1536                 break;
1537         }
1538
1539         return (error);
1540 }
1541
1542 static int
1543 ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1544 {
1545         struct ctl_be_block_filedata *file_data;
1546         struct ctl_lun_create_params *params;
1547         struct vattr                  vattr;
1548         int                           error;
1549
1550         error = 0;
1551         file_data = &be_lun->backend.file;
1552         params = &be_lun->params;
1553
1554         be_lun->dev_type = CTL_BE_BLOCK_FILE;
1555         be_lun->dispatch = ctl_be_block_dispatch_file;
1556         be_lun->lun_flush = ctl_be_block_flush_file;
1557
1558         error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
1559         if (error != 0) {
1560                 snprintf(req->error_str, sizeof(req->error_str),
1561                          "error calling VOP_GETATTR() for file %s",
1562                          be_lun->dev_path);
1563                 return (error);
1564         }
1565
1566         /*
1567          * Verify that we have the ability to upgrade to exclusive
1568          * access on this file so we can trap errors at open instead
1569          * of reporting them during first access.
1570          */
1571         if (VOP_ISLOCKED(be_lun->vn) != LK_EXCLUSIVE) {
1572                 vn_lock(be_lun->vn, LK_UPGRADE | LK_RETRY);
1573                 if (be_lun->vn->v_iflag & VI_DOOMED) {
1574                         error = EBADF;
1575                         snprintf(req->error_str, sizeof(req->error_str),
1576                                  "error locking file %s", be_lun->dev_path);
1577                         return (error);
1578                 }
1579         }
1580
1581
1582         file_data->cred = crhold(curthread->td_ucred);
1583         if (params->lun_size_bytes != 0)
1584                 be_lun->size_bytes = params->lun_size_bytes;
1585         else
1586                 be_lun->size_bytes = vattr.va_size;
1587         /*
1588          * We set the multi thread flag for file operations because all
1589          * filesystems (in theory) are capable of allowing multiple readers
1590          * of a file at once.  So we want to get the maximum possible
1591          * concurrency.
1592          */
1593         be_lun->flags |= CTL_BE_BLOCK_LUN_MULTI_THREAD;
1594
1595         /*
1596          * XXX KDM vattr.va_blocksize may be larger than 512 bytes here.
1597          * With ZFS, it is 131072 bytes.  Block sizes that large don't work
1598          * with disklabel and UFS on FreeBSD at least.  Large block sizes
1599          * may not work with other OSes as well.  So just export a sector
1600          * size of 512 bytes, which should work with any OS or
1601          * application.  Since our backing is a file, any block size will
1602          * work fine for the backing store.
1603          */
1604 #if 0
1605         be_lun->blocksize= vattr.va_blocksize;
1606 #endif
1607         if (params->blocksize_bytes != 0)
1608                 be_lun->blocksize = params->blocksize_bytes;
1609         else
1610                 be_lun->blocksize = 512;
1611
1612         /*
1613          * Sanity check.  The media size has to be at least one
1614          * sector long.
1615          */
1616         if (be_lun->size_bytes < be_lun->blocksize) {
1617                 error = EINVAL;
1618                 snprintf(req->error_str, sizeof(req->error_str),
1619                          "file %s size %ju < block size %u", be_lun->dev_path,
1620                          (uintmax_t)be_lun->size_bytes, be_lun->blocksize);
1621         }
1622         return (error);
1623 }
1624
1625 static int
1626 ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1627 {
1628         struct ctl_lun_create_params *params;
1629         struct vattr                  vattr;
1630         struct cdev                  *dev;
1631         struct cdevsw                *devsw;
1632         int                           error;
1633         off_t                         ps, pss, po, pos;
1634
1635         params = &be_lun->params;
1636
1637         be_lun->dev_type = CTL_BE_BLOCK_DEV;
1638         be_lun->backend.dev.cdev = be_lun->vn->v_rdev;
1639         be_lun->backend.dev.csw = dev_refthread(be_lun->backend.dev.cdev,
1640                                              &be_lun->backend.dev.dev_ref);
1641         if (be_lun->backend.dev.csw == NULL)
1642                 panic("Unable to retrieve device switch");
1643         if (strcmp(be_lun->backend.dev.csw->d_name, "zvol") == 0)
1644                 be_lun->dispatch = ctl_be_block_dispatch_zvol;
1645         else
1646                 be_lun->dispatch = ctl_be_block_dispatch_dev;
1647         be_lun->lun_flush = ctl_be_block_flush_dev;
1648         be_lun->unmap = ctl_be_block_unmap_dev;
1649
1650         error = VOP_GETATTR(be_lun->vn, &vattr, NOCRED);
1651         if (error) {
1652                 snprintf(req->error_str, sizeof(req->error_str),
1653                          "error getting vnode attributes for device %s",
1654                          be_lun->dev_path);
1655                 return (error);
1656         }
1657
1658         dev = be_lun->vn->v_rdev;
1659         devsw = dev->si_devsw;
1660         if (!devsw->d_ioctl) {
1661                 snprintf(req->error_str, sizeof(req->error_str),
1662                          "no d_ioctl for device %s!",
1663                          be_lun->dev_path);
1664                 return (ENODEV);
1665         }
1666
1667         error = devsw->d_ioctl(dev, DIOCGSECTORSIZE,
1668                                (caddr_t)&be_lun->blocksize, FREAD,
1669                                curthread);
1670         if (error) {
1671                 snprintf(req->error_str, sizeof(req->error_str),
1672                          "error %d returned for DIOCGSECTORSIZE ioctl "
1673                          "on %s!", error, be_lun->dev_path);
1674                 return (error);
1675         }
1676
1677         /*
1678          * If the user has asked for a blocksize that is greater than the
1679          * backing device's blocksize, we can do it only if the blocksize
1680          * the user is asking for is an even multiple of the underlying
1681          * device's blocksize.
1682          */
1683         if ((params->blocksize_bytes != 0)
1684          && (params->blocksize_bytes > be_lun->blocksize)) {
1685                 uint32_t bs_multiple, tmp_blocksize;
1686
1687                 bs_multiple = params->blocksize_bytes / be_lun->blocksize;
1688
1689                 tmp_blocksize = bs_multiple * be_lun->blocksize;
1690
1691                 if (tmp_blocksize == params->blocksize_bytes) {
1692                         be_lun->blocksize = params->blocksize_bytes;
1693                 } else {
1694                         snprintf(req->error_str, sizeof(req->error_str),
1695                                  "requested blocksize %u is not an even "
1696                                  "multiple of backing device blocksize %u",
1697                                  params->blocksize_bytes,
1698                                  be_lun->blocksize);
1699                         return (EINVAL);
1700
1701                 }
1702         } else if ((params->blocksize_bytes != 0)
1703                 && (params->blocksize_bytes != be_lun->blocksize)) {
1704                 snprintf(req->error_str, sizeof(req->error_str),
1705                          "requested blocksize %u < backing device "
1706                          "blocksize %u", params->blocksize_bytes,
1707                          be_lun->blocksize);
1708                 return (EINVAL);
1709         }
1710
1711         error = devsw->d_ioctl(dev, DIOCGMEDIASIZE,
1712                                (caddr_t)&be_lun->size_bytes, FREAD,
1713                                curthread);
1714         if (error) {
1715                 snprintf(req->error_str, sizeof(req->error_str),
1716                          "error %d returned for DIOCGMEDIASIZE "
1717                          " ioctl on %s!", error,
1718                          be_lun->dev_path);
1719                 return (error);
1720         }
1721
1722         if (params->lun_size_bytes != 0) {
1723                 if (params->lun_size_bytes > be_lun->size_bytes) {
1724                         snprintf(req->error_str, sizeof(req->error_str),
1725                                  "requested LUN size %ju > backing device "
1726                                  "size %ju",
1727                                  (uintmax_t)params->lun_size_bytes,
1728                                  (uintmax_t)be_lun->size_bytes);
1729                         return (EINVAL);
1730                 }
1731
1732                 be_lun->size_bytes = params->lun_size_bytes;
1733         }
1734
1735         error = devsw->d_ioctl(dev, DIOCGSTRIPESIZE,
1736                                (caddr_t)&ps, FREAD, curthread);
1737         if (error)
1738                 ps = po = 0;
1739         else {
1740                 error = devsw->d_ioctl(dev, DIOCGSTRIPEOFFSET,
1741                                        (caddr_t)&po, FREAD, curthread);
1742                 if (error)
1743                         po = 0;
1744         }
1745         pss = ps / be_lun->blocksize;
1746         pos = po / be_lun->blocksize;
1747         if ((pss > 0) && (pss * be_lun->blocksize == ps) && (pss >= pos) &&
1748             ((pss & (pss - 1)) == 0) && (pos * be_lun->blocksize == po)) {
1749                 be_lun->pblockexp = fls(pss) - 1;
1750                 be_lun->pblockoff = (pss - pos) % pss;
1751         }
1752
1753         return (0);
1754 }
1755
1756 static int
1757 ctl_be_block_close(struct ctl_be_block_lun *be_lun)
1758 {
1759         DROP_GIANT();
1760         if (be_lun->vn) {
1761                 int flags = FREAD | FWRITE;
1762
1763                 switch (be_lun->dev_type) {
1764                 case CTL_BE_BLOCK_DEV:
1765                         if (be_lun->backend.dev.csw) {
1766                                 dev_relthread(be_lun->backend.dev.cdev,
1767                                               be_lun->backend.dev.dev_ref);
1768                                 be_lun->backend.dev.csw  = NULL;
1769                                 be_lun->backend.dev.cdev = NULL;
1770                         }
1771                         break;
1772                 case CTL_BE_BLOCK_FILE:
1773                         break;
1774                 case CTL_BE_BLOCK_NONE:
1775                         break;
1776                 default:
1777                         panic("Unexpected backend type.");
1778                         break;
1779                 }
1780
1781                 (void)vn_close(be_lun->vn, flags, NOCRED, curthread);
1782                 be_lun->vn = NULL;
1783
1784                 switch (be_lun->dev_type) {
1785                 case CTL_BE_BLOCK_DEV:
1786                         break;
1787                 case CTL_BE_BLOCK_FILE:
1788                         if (be_lun->backend.file.cred != NULL) {
1789                                 crfree(be_lun->backend.file.cred);
1790                                 be_lun->backend.file.cred = NULL;
1791                         }
1792                         break;
1793                 case CTL_BE_BLOCK_NONE:
1794                         break;
1795                 default:
1796                         panic("Unexpected backend type.");
1797                         break;
1798                 }
1799                 be_lun->dev_type = CTL_BE_BLOCK_NONE;
1800         }
1801         PICKUP_GIANT();
1802
1803         return (0);
1804 }
1805
1806 static int
1807 ctl_be_block_open(struct ctl_be_block_softc *softc,
1808                        struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1809 {
1810         struct nameidata nd;
1811         int              flags;
1812         int              error;
1813
1814         /*
1815          * XXX KDM allow a read-only option?
1816          */
1817         flags = FREAD | FWRITE;
1818         error = 0;
1819
1820         if (rootvnode == NULL) {
1821                 snprintf(req->error_str, sizeof(req->error_str),
1822                          "Root filesystem is not mounted");
1823                 return (1);
1824         }
1825
1826         if (!curthread->td_proc->p_fd->fd_cdir) {
1827                 curthread->td_proc->p_fd->fd_cdir = rootvnode;
1828                 VREF(rootvnode);
1829         }
1830         if (!curthread->td_proc->p_fd->fd_rdir) {
1831                 curthread->td_proc->p_fd->fd_rdir = rootvnode;
1832                 VREF(rootvnode);
1833         }
1834         if (!curthread->td_proc->p_fd->fd_jdir) {
1835                 curthread->td_proc->p_fd->fd_jdir = rootvnode;
1836                 VREF(rootvnode);
1837         }
1838
1839  again:
1840         NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread);
1841         error = vn_open(&nd, &flags, 0, NULL);
1842         if (error) {
1843                 /*
1844                  * This is the only reasonable guess we can make as far as
1845                  * path if the user doesn't give us a fully qualified path.
1846                  * If they want to specify a file, they need to specify the
1847                  * full path.
1848                  */
1849                 if (be_lun->dev_path[0] != '/') {
1850                         char *dev_path = "/dev/";
1851                         char *dev_name;
1852
1853                         /* Try adding device path at beginning of name */
1854                         dev_name = malloc(strlen(be_lun->dev_path)
1855                                         + strlen(dev_path) + 1,
1856                                           M_CTLBLK, M_WAITOK);
1857                         if (dev_name) {
1858                                 sprintf(dev_name, "%s%s", dev_path,
1859                                         be_lun->dev_path);
1860                                 free(be_lun->dev_path, M_CTLBLK);
1861                                 be_lun->dev_path = dev_name;
1862                                 goto again;
1863                         }
1864                 }
1865                 snprintf(req->error_str, sizeof(req->error_str),
1866                     "error opening %s: %d", be_lun->dev_path, error);
1867                 return (error);
1868         }
1869
1870         NDFREE(&nd, NDF_ONLY_PNBUF);
1871
1872         be_lun->vn = nd.ni_vp;
1873
1874         /* We only support disks and files. */
1875         if (vn_isdisk(be_lun->vn, &error)) {
1876                 error = ctl_be_block_open_dev(be_lun, req);
1877         } else if (be_lun->vn->v_type == VREG) {
1878                 error = ctl_be_block_open_file(be_lun, req);
1879         } else {
1880                 error = EINVAL;
1881                 snprintf(req->error_str, sizeof(req->error_str),
1882                          "%s is not a disk or plain file", be_lun->dev_path);
1883         }
1884         VOP_UNLOCK(be_lun->vn, 0);
1885
1886         if (error != 0) {
1887                 ctl_be_block_close(be_lun);
1888                 return (error);
1889         }
1890
1891         be_lun->blocksize_shift = fls(be_lun->blocksize) - 1;
1892         be_lun->size_blocks = be_lun->size_bytes >> be_lun->blocksize_shift;
1893
1894         return (0);
1895 }
1896
1897 static int
1898 ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
1899 {
1900         struct ctl_be_block_lun *be_lun;
1901         struct ctl_lun_create_params *params;
1902         char num_thread_str[16];
1903         char tmpstr[32];
1904         char *value;
1905         int retval, num_threads, unmap;
1906         int tmp_num_threads;
1907
1908         params = &req->reqdata.create;
1909         retval = 0;
1910         req->status = CTL_LUN_OK;
1911
1912         num_threads = cbb_num_threads;
1913
1914         be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);
1915
1916         be_lun->params = req->reqdata.create;
1917         be_lun->softc = softc;
1918         STAILQ_INIT(&be_lun->input_queue);
1919         STAILQ_INIT(&be_lun->config_write_queue);
1920         STAILQ_INIT(&be_lun->datamove_queue);
1921         sprintf(be_lun->lunname, "cblk%d", softc->num_luns);
1922         mtx_init(&be_lun->io_lock, "cblk io lock", NULL, MTX_DEF);
1923         mtx_init(&be_lun->queue_lock, "cblk queue lock", NULL, MTX_DEF);
1924         ctl_init_opts(&be_lun->ctl_be_lun.options,
1925             req->num_be_args, req->kern_be_args);
1926
1927         be_lun->lun_zone = uma_zcreate(be_lun->lunname, CTLBLK_MAX_SEG,
1928             NULL, NULL, NULL, NULL, /*align*/ 0, /*flags*/0);
1929
1930         if (be_lun->lun_zone == NULL) {
1931                 snprintf(req->error_str, sizeof(req->error_str),
1932                          "error allocating UMA zone");
1933                 goto bailout_error;
1934         }
1935
1936         if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
1937                 be_lun->ctl_be_lun.lun_type = params->device_type;
1938         else
1939                 be_lun->ctl_be_lun.lun_type = T_DIRECT;
1940
1941         if (be_lun->ctl_be_lun.lun_type == T_DIRECT) {
1942                 value = ctl_get_opt(&be_lun->ctl_be_lun.options, "file");
1943                 if (value == NULL) {
1944                         snprintf(req->error_str, sizeof(req->error_str),
1945                                  "no file argument specified");
1946                         goto bailout_error;
1947                 }
1948                 be_lun->dev_path = strdup(value, M_CTLBLK);
1949                 be_lun->blocksize = 512;
1950                 be_lun->blocksize_shift = fls(be_lun->blocksize) - 1;
1951
1952                 retval = ctl_be_block_open(softc, be_lun, req);
1953                 if (retval != 0) {
1954                         retval = 0;
1955                         req->status = CTL_LUN_WARNING;
1956                 }
1957         } else {
1958                 /*
1959                  * For processor devices, we don't have any size.
1960                  */
1961                 be_lun->blocksize = 0;
1962                 be_lun->pblockexp = 0;
1963                 be_lun->pblockoff = 0;
1964                 be_lun->size_blocks = 0;
1965                 be_lun->size_bytes = 0;
1966                 be_lun->ctl_be_lun.maxlba = 0;
1967
1968                 /*
1969                  * Default to just 1 thread for processor devices.
1970                  */
1971                 num_threads = 1;
1972         }
1973
1974         /*
1975          * XXX This searching loop might be refactored to be combined with
1976          * the loop above,
1977          */
1978         value = ctl_get_opt(&be_lun->ctl_be_lun.options, "num_threads");
1979         if (value != NULL) {
1980                 tmp_num_threads = strtol(value, NULL, 0);
1981
1982                 /*
1983                  * We don't let the user specify less than one
1984                  * thread, but hope he's clueful enough not to
1985                  * specify 1000 threads.
1986                  */
1987                 if (tmp_num_threads < 1) {
1988                         snprintf(req->error_str, sizeof(req->error_str),
1989                                  "invalid number of threads %s",
1990                                  num_thread_str);
1991                         goto bailout_error;
1992                 }
1993                 num_threads = tmp_num_threads;
1994         }
1995         unmap = 0;
1996         value = ctl_get_opt(&be_lun->ctl_be_lun.options, "unmap");
1997         if (value != NULL && strcmp(value, "on") == 0)
1998                 unmap = 1;
1999
2000         be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED;
2001         be_lun->ctl_be_lun.flags = CTL_LUN_FLAG_PRIMARY;
2002         if (be_lun->vn == NULL)
2003                 be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_OFFLINE;
2004         if (unmap)
2005                 be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_UNMAP;
2006         be_lun->ctl_be_lun.be_lun = be_lun;
2007         be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ?
2008             0 : (be_lun->size_blocks - 1);
2009         be_lun->ctl_be_lun.blocksize = be_lun->blocksize;
2010         be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp;
2011         be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff;
2012         if (be_lun->dispatch == ctl_be_block_dispatch_zvol &&
2013             be_lun->blocksize != 0)
2014                 be_lun->ctl_be_lun.atomicblock = CTLBLK_MAX_IO_SIZE /
2015                     be_lun->blocksize;
2016         /* Tell the user the blocksize we ended up using */
2017         params->lun_size_bytes = be_lun->size_bytes;
2018         params->blocksize_bytes = be_lun->blocksize;
2019         if (params->flags & CTL_LUN_FLAG_ID_REQ) {
2020                 be_lun->ctl_be_lun.req_lun_id = params->req_lun_id;
2021                 be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_ID_REQ;
2022         } else
2023                 be_lun->ctl_be_lun.req_lun_id = 0;
2024
2025         be_lun->ctl_be_lun.lun_shutdown = ctl_be_block_lun_shutdown;
2026         be_lun->ctl_be_lun.lun_config_status =
2027                 ctl_be_block_lun_config_status;
2028         be_lun->ctl_be_lun.be = &ctl_be_block_driver;
2029
2030         if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) {
2031                 snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%4d",
2032                          softc->num_luns);
2033                 strncpy((char *)be_lun->ctl_be_lun.serial_num, tmpstr,
2034                         ctl_min(sizeof(be_lun->ctl_be_lun.serial_num),
2035                         sizeof(tmpstr)));
2036
2037                 /* Tell the user what we used for a serial number */
2038                 strncpy((char *)params->serial_num, tmpstr,
2039                         ctl_min(sizeof(params->serial_num), sizeof(tmpstr)));
2040         } else {
2041                 strncpy((char *)be_lun->ctl_be_lun.serial_num,
2042                         params->serial_num,
2043                         ctl_min(sizeof(be_lun->ctl_be_lun.serial_num),
2044                         sizeof(params->serial_num)));
2045         }
2046         if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) {
2047                 snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%4d", softc->num_luns);
2048                 strncpy((char *)be_lun->ctl_be_lun.device_id, tmpstr,
2049                         ctl_min(sizeof(be_lun->ctl_be_lun.device_id),
2050                         sizeof(tmpstr)));
2051
2052                 /* Tell the user what we used for a device ID */
2053                 strncpy((char *)params->device_id, tmpstr,
2054                         ctl_min(sizeof(params->device_id), sizeof(tmpstr)));
2055         } else {
2056                 strncpy((char *)be_lun->ctl_be_lun.device_id,
2057                         params->device_id,
2058                         ctl_min(sizeof(be_lun->ctl_be_lun.device_id),
2059                                 sizeof(params->device_id)));
2060         }
2061
2062         TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);
2063
2064         be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK,
2065             taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);
2066
2067         if (be_lun->io_taskqueue == NULL) {
2068                 snprintf(req->error_str, sizeof(req->error_str),
2069                          "unable to create taskqueue");
2070                 goto bailout_error;
2071         }
2072
2073         /*
2074          * Note that we start the same number of threads by default for
2075          * both the file case and the block device case.  For the file
2076          * case, we need multiple threads to allow concurrency, because the
2077          * vnode interface is designed to be a blocking interface.  For the
2078          * block device case, ZFS zvols at least will block the caller's
2079          * context in many instances, and so we need multiple threads to
2080          * overcome that problem.  Other block devices don't need as many
2081          * threads, but they shouldn't cause too many problems.
2082          *
2083          * If the user wants to just have a single thread for a block
2084          * device, he can specify that when the LUN is created, or change
2085          * the tunable/sysctl to alter the default number of threads.
2086          */
2087         retval = taskqueue_start_threads(&be_lun->io_taskqueue,
2088                                          /*num threads*/num_threads,
2089                                          /*priority*/PWAIT,
2090                                          /*thread name*/
2091                                          "%s taskq", be_lun->lunname);
2092
2093         if (retval != 0)
2094                 goto bailout_error;
2095
2096         be_lun->num_threads = num_threads;
2097
2098         mtx_lock(&softc->lock);
2099         softc->num_luns++;
2100         STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links);
2101
2102         mtx_unlock(&softc->lock);
2103
2104         retval = ctl_add_lun(&be_lun->ctl_be_lun);
2105         if (retval != 0) {
2106                 mtx_lock(&softc->lock);
2107                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2108                               links);
2109                 softc->num_luns--;
2110                 mtx_unlock(&softc->lock);
2111                 snprintf(req->error_str, sizeof(req->error_str),
2112                          "ctl_add_lun() returned error %d, see dmesg for "
2113                          "details", retval);
2114                 retval = 0;
2115                 goto bailout_error;
2116         }
2117
2118         mtx_lock(&softc->lock);
2119
2120         /*
2121          * Tell the config_status routine that we're waiting so it won't
2122          * clean up the LUN in the event of an error.
2123          */
2124         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2125
2126         while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {
2127                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2128                 if (retval == EINTR)
2129                         break;
2130         }
2131         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2132
2133         if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) {
2134                 snprintf(req->error_str, sizeof(req->error_str),
2135                          "LUN configuration error, see dmesg for details");
2136                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
2137                               links);
2138                 softc->num_luns--;
2139                 mtx_unlock(&softc->lock);
2140                 goto bailout_error;
2141         } else {
2142                 params->req_lun_id = be_lun->ctl_be_lun.lun_id;
2143         }
2144
2145         mtx_unlock(&softc->lock);
2146
2147         be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id,
2148                                                be_lun->blocksize,
2149                                                DEVSTAT_ALL_SUPPORTED,
2150                                                be_lun->ctl_be_lun.lun_type
2151                                                | DEVSTAT_TYPE_IF_OTHER,
2152                                                DEVSTAT_PRIORITY_OTHER);
2153
2154         return (retval);
2155
2156 bailout_error:
2157         req->status = CTL_LUN_ERROR;
2158
2159         if (be_lun->io_taskqueue != NULL)
2160                 taskqueue_free(be_lun->io_taskqueue);
2161         ctl_be_block_close(be_lun);
2162         if (be_lun->dev_path != NULL)
2163                 free(be_lun->dev_path, M_CTLBLK);
2164         if (be_lun->lun_zone != NULL)
2165                 uma_zdestroy(be_lun->lun_zone);
2166         ctl_free_opts(&be_lun->ctl_be_lun.options);
2167         mtx_destroy(&be_lun->queue_lock);
2168         mtx_destroy(&be_lun->io_lock);
2169         free(be_lun, M_CTLBLK);
2170
2171         return (retval);
2172 }
2173
2174 static int
2175 ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2176 {
2177         struct ctl_lun_rm_params *params;
2178         struct ctl_be_block_lun *be_lun;
2179         int retval;
2180
2181         params = &req->reqdata.rm;
2182
2183         mtx_lock(&softc->lock);
2184
2185         be_lun = NULL;
2186
2187         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2188                 if (be_lun->ctl_be_lun.lun_id == params->lun_id)
2189                         break;
2190         }
2191         mtx_unlock(&softc->lock);
2192
2193         if (be_lun == NULL) {
2194                 snprintf(req->error_str, sizeof(req->error_str),
2195                          "LUN %u is not managed by the block backend",
2196                          params->lun_id);
2197                 goto bailout_error;
2198         }
2199
2200         retval = ctl_disable_lun(&be_lun->ctl_be_lun);
2201
2202         if (retval != 0) {
2203                 snprintf(req->error_str, sizeof(req->error_str),
2204                          "error %d returned from ctl_disable_lun() for "
2205                          "LUN %d", retval, params->lun_id);
2206                 goto bailout_error;
2207
2208         }
2209
2210         retval = ctl_invalidate_lun(&be_lun->ctl_be_lun);
2211         if (retval != 0) {
2212                 snprintf(req->error_str, sizeof(req->error_str),
2213                          "error %d returned from ctl_invalidate_lun() for "
2214                          "LUN %d", retval, params->lun_id);
2215                 goto bailout_error;
2216         }
2217
2218         mtx_lock(&softc->lock);
2219
2220         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
2221
2222         while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2223                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
2224                 if (retval == EINTR)
2225                         break;
2226         }
2227
2228         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
2229
2230         if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
2231                 snprintf(req->error_str, sizeof(req->error_str),
2232                          "interrupted waiting for LUN to be freed");
2233                 mtx_unlock(&softc->lock);
2234                 goto bailout_error;
2235         }
2236
2237         STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links);
2238
2239         softc->num_luns--;
2240         mtx_unlock(&softc->lock);
2241
2242         taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task);
2243
2244         taskqueue_free(be_lun->io_taskqueue);
2245
2246         ctl_be_block_close(be_lun);
2247
2248         if (be_lun->disk_stats != NULL)
2249                 devstat_remove_entry(be_lun->disk_stats);
2250
2251         uma_zdestroy(be_lun->lun_zone);
2252
2253         ctl_free_opts(&be_lun->ctl_be_lun.options);
2254         free(be_lun->dev_path, M_CTLBLK);
2255         mtx_destroy(&be_lun->queue_lock);
2256         mtx_destroy(&be_lun->io_lock);
2257         free(be_lun, M_CTLBLK);
2258
2259         req->status = CTL_LUN_OK;
2260
2261         return (0);
2262
2263 bailout_error:
2264
2265         req->status = CTL_LUN_ERROR;
2266
2267         return (0);
2268 }
2269
2270 static int
2271 ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
2272                          struct ctl_lun_req *req)
2273 {
2274         struct vattr vattr;
2275         int error;
2276         struct ctl_lun_create_params *params = &be_lun->params;
2277
2278         if (params->lun_size_bytes != 0) {
2279                 be_lun->size_bytes = params->lun_size_bytes;
2280         } else  {
2281                 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
2282                 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
2283                 VOP_UNLOCK(be_lun->vn, 0);
2284                 if (error != 0) {
2285                         snprintf(req->error_str, sizeof(req->error_str),
2286                                  "error calling VOP_GETATTR() for file %s",
2287                                  be_lun->dev_path);
2288                         return (error);
2289                 }
2290
2291                 be_lun->size_bytes = vattr.va_size;
2292         }
2293
2294         return (0);
2295 }
2296
2297 static int
2298 ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
2299                         struct ctl_lun_req *req)
2300 {
2301         struct ctl_be_block_devdata *dev_data;
2302         int error;
2303         struct ctl_lun_create_params *params = &be_lun->params;
2304         uint64_t size_bytes;
2305
2306         dev_data = &be_lun->backend.dev;
2307         if (!dev_data->csw->d_ioctl) {
2308                 snprintf(req->error_str, sizeof(req->error_str),
2309                          "no d_ioctl for device %s!", be_lun->dev_path);
2310                 return (ENODEV);
2311         }
2312
2313         error = dev_data->csw->d_ioctl(dev_data->cdev, DIOCGMEDIASIZE,
2314                                (caddr_t)&size_bytes, FREAD,
2315                                curthread);
2316         if (error) {
2317                 snprintf(req->error_str, sizeof(req->error_str),
2318                          "error %d returned for DIOCGMEDIASIZE ioctl "
2319                          "on %s!", error, be_lun->dev_path);
2320                 return (error);
2321         }
2322
2323         if (params->lun_size_bytes != 0) {
2324                 if (params->lun_size_bytes > size_bytes) {
2325                         snprintf(req->error_str, sizeof(req->error_str),
2326                                  "requested LUN size %ju > backing device "
2327                                  "size %ju",
2328                                  (uintmax_t)params->lun_size_bytes,
2329                                  (uintmax_t)size_bytes);
2330                         return (EINVAL);
2331                 }
2332
2333                 be_lun->size_bytes = params->lun_size_bytes;
2334         } else {
2335                 be_lun->size_bytes = size_bytes;
2336         }
2337
2338         return (0);
2339 }
2340
2341 static int
2342 ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2343 {
2344         struct ctl_lun_modify_params *params;
2345         struct ctl_be_block_lun *be_lun;
2346         uint64_t oldsize;
2347         int error;
2348
2349         params = &req->reqdata.modify;
2350
2351         mtx_lock(&softc->lock);
2352         be_lun = NULL;
2353         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2354                 if (be_lun->ctl_be_lun.lun_id == params->lun_id)
2355                         break;
2356         }
2357         mtx_unlock(&softc->lock);
2358
2359         if (be_lun == NULL) {
2360                 snprintf(req->error_str, sizeof(req->error_str),
2361                          "LUN %u is not managed by the block backend",
2362                          params->lun_id);
2363                 goto bailout_error;
2364         }
2365
2366         be_lun->params.lun_size_bytes = params->lun_size_bytes;
2367
2368         oldsize = be_lun->size_blocks;
2369         if (be_lun->vn == NULL)
2370                 error = ctl_be_block_open(softc, be_lun, req);
2371         else if (be_lun->vn->v_type == VREG)
2372                 error = ctl_be_block_modify_file(be_lun, req);
2373         else
2374                 error = ctl_be_block_modify_dev(be_lun, req);
2375
2376         if (error == 0 && be_lun->size_blocks != oldsize) {
2377                 be_lun->size_blocks = be_lun->size_bytes >>
2378                     be_lun->blocksize_shift;
2379
2380                 /*
2381                  * The maximum LBA is the size - 1.
2382                  *
2383                  * XXX: Note that this field is being updated without locking,
2384                  *      which might cause problems on 32-bit architectures.
2385                  */
2386                 be_lun->ctl_be_lun.maxlba = (be_lun->size_blocks == 0) ?
2387                     0 : (be_lun->size_blocks - 1);
2388                 be_lun->ctl_be_lun.blocksize = be_lun->blocksize;
2389                 be_lun->ctl_be_lun.pblockexp = be_lun->pblockexp;
2390                 be_lun->ctl_be_lun.pblockoff = be_lun->pblockoff;
2391                 if (be_lun->dispatch == ctl_be_block_dispatch_zvol &&
2392                     be_lun->blocksize != 0)
2393                         be_lun->ctl_be_lun.atomicblock = CTLBLK_MAX_IO_SIZE /
2394                             be_lun->blocksize;
2395                 ctl_lun_capacity_changed(&be_lun->ctl_be_lun);
2396                 if (oldsize == 0 && be_lun->size_blocks != 0)
2397                         ctl_lun_online(&be_lun->ctl_be_lun);
2398         }
2399
2400         /* Tell the user the exact size we ended up using */
2401         params->lun_size_bytes = be_lun->size_bytes;
2402
2403         req->status = error ? CTL_LUN_WARNING : CTL_LUN_OK;
2404
2405         return (0);
2406
2407 bailout_error:
2408         req->status = CTL_LUN_ERROR;
2409
2410         return (0);
2411 }
2412
2413 static void
2414 ctl_be_block_lun_shutdown(void *be_lun)
2415 {
2416         struct ctl_be_block_lun *lun;
2417         struct ctl_be_block_softc *softc;
2418
2419         lun = (struct ctl_be_block_lun *)be_lun;
2420
2421         softc = lun->softc;
2422
2423         mtx_lock(&softc->lock);
2424         lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;
2425         if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2426                 wakeup(lun);
2427         mtx_unlock(&softc->lock);
2428
2429 }
2430
2431 static void
2432 ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status)
2433 {
2434         struct ctl_be_block_lun *lun;
2435         struct ctl_be_block_softc *softc;
2436
2437         lun = (struct ctl_be_block_lun *)be_lun;
2438         softc = lun->softc;
2439
2440         if (status == CTL_LUN_CONFIG_OK) {
2441                 mtx_lock(&softc->lock);
2442                 lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2443                 if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2444                         wakeup(lun);
2445                 mtx_unlock(&softc->lock);
2446
2447                 /*
2448                  * We successfully added the LUN, attempt to enable it.
2449                  */
2450                 if (ctl_enable_lun(&lun->ctl_be_lun) != 0) {
2451                         printf("%s: ctl_enable_lun() failed!\n", __func__);
2452                         if (ctl_invalidate_lun(&lun->ctl_be_lun) != 0) {
2453                                 printf("%s: ctl_invalidate_lun() failed!\n",
2454                                        __func__);
2455                         }
2456                 }
2457
2458                 return;
2459         }
2460
2461
2462         mtx_lock(&softc->lock);
2463         lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2464         lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR;
2465         wakeup(lun);
2466         mtx_unlock(&softc->lock);
2467 }
2468
2469
2470 static int
2471 ctl_be_block_config_write(union ctl_io *io)
2472 {
2473         struct ctl_be_block_lun *be_lun;
2474         struct ctl_be_lun *ctl_be_lun;
2475         int retval;
2476
2477         retval = 0;
2478
2479         DPRINTF("entered\n");
2480
2481         ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
2482                 CTL_PRIV_BACKEND_LUN].ptr;
2483         be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun;
2484
2485         switch (io->scsiio.cdb[0]) {
2486         case SYNCHRONIZE_CACHE:
2487         case SYNCHRONIZE_CACHE_16:
2488         case WRITE_SAME_10:
2489         case WRITE_SAME_16:
2490         case UNMAP:
2491                 /*
2492                  * The upper level CTL code will filter out any CDBs with
2493                  * the immediate bit set and return the proper error.
2494                  *
2495                  * We don't really need to worry about what LBA range the
2496                  * user asked to be synced out.  When they issue a sync
2497                  * cache command, we'll sync out the whole thing.
2498                  */
2499                 mtx_lock(&be_lun->queue_lock);
2500                 STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,
2501                                    links);
2502                 mtx_unlock(&be_lun->queue_lock);
2503                 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
2504                 break;
2505         case START_STOP_UNIT: {
2506                 struct scsi_start_stop_unit *cdb;
2507
2508                 cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
2509
2510                 if (cdb->how & SSS_START)
2511                         retval = ctl_start_lun(ctl_be_lun);
2512                 else {
2513                         retval = ctl_stop_lun(ctl_be_lun);
2514                         /*
2515                          * XXX KDM Copan-specific offline behavior.
2516                          * Figure out a reasonable way to port this?
2517                          */
2518 #ifdef NEEDTOPORT
2519                         if ((retval == 0)
2520                          && (cdb->byte2 & SSS_ONOFFLINE))
2521                                 retval = ctl_lun_offline(ctl_be_lun);
2522 #endif
2523                 }
2524
2525                 /*
2526                  * In general, the above routines should not fail.  They
2527                  * just set state for the LUN.  So we've got something
2528                  * pretty wrong here if we can't start or stop the LUN.
2529                  */
2530                 if (retval != 0) {
2531                         ctl_set_internal_failure(&io->scsiio,
2532                                                  /*sks_valid*/ 1,
2533                                                  /*retry_count*/ 0xf051);
2534                         retval = CTL_RETVAL_COMPLETE;
2535                 } else {
2536                         ctl_set_success(&io->scsiio);
2537                 }
2538                 ctl_config_write_done(io);
2539                 break;
2540         }
2541         default:
2542                 ctl_set_invalid_opcode(&io->scsiio);
2543                 ctl_config_write_done(io);
2544                 retval = CTL_RETVAL_COMPLETE;
2545                 break;
2546         }
2547
2548         return (retval);
2549
2550 }
2551
2552 static int
2553 ctl_be_block_config_read(union ctl_io *io)
2554 {
2555         return (0);
2556 }
2557
2558 static int
2559 ctl_be_block_lun_info(void *be_lun, struct sbuf *sb)
2560 {
2561         struct ctl_be_block_lun *lun;
2562         int retval;
2563
2564         lun = (struct ctl_be_block_lun *)be_lun;
2565         retval = 0;
2566
2567         retval = sbuf_printf(sb, "\t<num_threads>");
2568
2569         if (retval != 0)
2570                 goto bailout;
2571
2572         retval = sbuf_printf(sb, "%d", lun->num_threads);
2573
2574         if (retval != 0)
2575                 goto bailout;
2576
2577         retval = sbuf_printf(sb, "</num_threads>\n");
2578
2579 bailout:
2580
2581         return (retval);
2582 }
2583
2584 int
2585 ctl_be_block_init(void)
2586 {
2587         struct ctl_be_block_softc *softc;
2588         int retval;
2589
2590         softc = &backend_block_softc;
2591         retval = 0;
2592
2593         mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF);
2594         beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io),
2595             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
2596         STAILQ_INIT(&softc->disk_list);
2597         STAILQ_INIT(&softc->lun_list);
2598
2599         return (retval);
2600 }