sys/cam/ctl/ctl_backend_block.c

   1 /*-
   2  * Copyright (c) 2003 Silicon Graphics International Corp.
   3  * Copyright (c) 2009-2011 Spectra Logic Corporation
   4  * Copyright (c) 2012 The FreeBSD Foundation
   5  * All rights reserved.
   6  *
   7  * Portions of this software were developed by Edward Tomasz Napierala
   8  * under sponsorship from the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions, and the following disclaimer,
  15  *    without modification.
  16  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  17  *    substantially similar to the "NO WARRANTY" disclaimer below
  18  *    ("Disclaimer") and any redistribution must be conditioned upon
  19  *    including a substantially similar Disclaimer requirement for further
  20  *    binary redistribution.
  21  *
  22  * NO WARRANTY
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  31  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  32  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  33  * POSSIBILITY OF SUCH DAMAGES.
  34  *
  35  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_backend_block.c#5 $
  36  */
  37 /*
  38  * CAM Target Layer driver backend for block devices.
  39  *
  40  * Author: Ken Merry <ken@FreeBSD.org>
  41  */
  42 #include <sys/cdefs.h>
  43 __FBSDID("$FreeBSD$");
  44
  45 #include <opt_kdtrace.h>
  46
  47 #include <sys/param.h>
  48 #include <sys/systm.h>
  49 #include <sys/kernel.h>
  50 #include <sys/types.h>
  51 #include <sys/kthread.h>
  52 #include <sys/bio.h>
  53 #include <sys/fcntl.h>
  54 #include <sys/lock.h>
  55 #include <sys/mutex.h>
  56 #include <sys/condvar.h>
  57 #include <sys/malloc.h>
  58 #include <sys/conf.h>
  59 #include <sys/ioccom.h>
  60 #include <sys/queue.h>
  61 #include <sys/sbuf.h>
  62 #include <sys/endian.h>
  63 #include <sys/uio.h>
  64 #include <sys/buf.h>
  65 #include <sys/taskqueue.h>
  66 #include <sys/vnode.h>
  67 #include <sys/namei.h>
  68 #include <sys/mount.h>
  69 #include <sys/disk.h>
  70 #include <sys/fcntl.h>
  71 #include <sys/filedesc.h>
  72 #include <sys/proc.h>
  73 #include <sys/pcpu.h>
  74 #include <sys/module.h>
  75 #include <sys/sdt.h>
  76 #include <sys/devicestat.h>
  77 #include <sys/sysctl.h>
  78
  79 #include <geom/geom.h>
  80
  81 #include <cam/cam.h>
  82 #include <cam/scsi/scsi_all.h>
  83 #include <cam/scsi/scsi_da.h>
  84 #include <cam/ctl/ctl_io.h>
  85 #include <cam/ctl/ctl.h>
  86 #include <cam/ctl/ctl_backend.h>
  87 #include <cam/ctl/ctl_frontend_internal.h>
  88 #include <cam/ctl/ctl_ioctl.h>
  89 #include <cam/ctl/ctl_scsi_all.h>
  90 #include <cam/ctl/ctl_error.h>
  91
  92 /*
  93  * The idea here is that we'll allocate enough S/G space to hold a 16MB
  94  * I/O.  If we get an I/O larger than that, we'll reject it.
  95  */
  96 #define CTLBLK_MAX_IO_SIZE      (16 * 1024 * 1024)
  97 #define CTLBLK_MAX_SEGS         (CTLBLK_MAX_IO_SIZE / MAXPHYS) + 1
  98
  99 #ifdef CTLBLK_DEBUG
 100 #define DPRINTF(fmt, args...) \
 101     printf("cbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args)
 102 #else
 103 #define DPRINTF(fmt, args...) do {} while(0)
 104 #endif
 105
 106 SDT_PROVIDER_DEFINE(cbb);
 107
 108 typedef enum {
 109         CTL_BE_BLOCK_LUN_UNCONFIGURED   = 0x01,
 110         CTL_BE_BLOCK_LUN_CONFIG_ERR     = 0x02,
 111         CTL_BE_BLOCK_LUN_WAITING        = 0x04,
 112         CTL_BE_BLOCK_LUN_MULTI_THREAD   = 0x08
 113 } ctl_be_block_lun_flags;
 114
 115 typedef enum {
 116         CTL_BE_BLOCK_NONE,
 117         CTL_BE_BLOCK_DEV,
 118         CTL_BE_BLOCK_FILE
 119 } ctl_be_block_type;
 120
 121 struct ctl_be_block_devdata {
 122         struct cdev *cdev;
 123         struct cdevsw *csw;
 124         int dev_ref;
 125 };
 126
 127 struct ctl_be_block_filedata {
 128         struct ucred *cred;
 129 };
 130
 131 union ctl_be_block_bedata {
 132         struct ctl_be_block_devdata dev;
 133         struct ctl_be_block_filedata file;
 134 };
 135
 136 struct ctl_be_block_io;
 137 struct ctl_be_block_lun;
 138
 139 typedef void (*cbb_dispatch_t)(struct ctl_be_block_lun *be_lun,
 140                                struct ctl_be_block_io *beio);
 141
 142 /*
 143  * Backend LUN structure.  There is a 1:1 mapping between a block device
 144  * and a backend block LUN, and between a backend block LUN and a CTL LUN.
 145  */
 146 struct ctl_be_block_lun {
 147         struct ctl_block_disk *disk;
 148         char lunname[32];
 149         char *dev_path;
 150         ctl_be_block_type dev_type;
 151         struct vnode *vn;
 152         union ctl_be_block_bedata backend;
 153         cbb_dispatch_t dispatch;
 154         cbb_dispatch_t lun_flush;
 155         struct mtx lock;
 156         uma_zone_t lun_zone;
 157         uint64_t size_blocks;
 158         uint64_t size_bytes;
 159         uint32_t blocksize;
 160         int blocksize_shift;
 161         struct ctl_be_block_softc *softc;
 162         struct devstat *disk_stats;
 163         ctl_be_block_lun_flags flags;
 164         STAILQ_ENTRY(ctl_be_block_lun) links;
 165         struct ctl_be_lun ctl_be_lun;
 166         struct taskqueue *io_taskqueue;
 167         struct task io_task;
 168         int num_threads;
 169         STAILQ_HEAD(, ctl_io_hdr) input_queue;
 170         STAILQ_HEAD(, ctl_io_hdr) config_write_queue;
 171         STAILQ_HEAD(, ctl_io_hdr) datamove_queue;
 172 };
 173
 174 /*
 175  * Overall softc structure for the block backend module.
 176  */
 177 struct ctl_be_block_softc {
 178         STAILQ_HEAD(, ctl_be_block_io)   beio_free_queue;
 179         struct mtx                       lock;
 180         int                              prealloc_beio;
 181         int                              num_disks;
 182         STAILQ_HEAD(, ctl_block_disk)    disk_list;
 183         int                              num_luns;
 184         STAILQ_HEAD(, ctl_be_block_lun)  lun_list;
 185 };
 186
 187 static struct ctl_be_block_softc backend_block_softc;
 188
 189 /*
 190  * Per-I/O information.
 191  */
 192 struct ctl_be_block_io {
 193         union ctl_io                    *io;
 194         struct ctl_sg_entry             sg_segs[CTLBLK_MAX_SEGS];
 195         struct iovec                    xiovecs[CTLBLK_MAX_SEGS];
 196         int                             bio_cmd;
 197         int                             bio_flags;
 198         int                             num_segs;
 199         int                             num_bios_sent;
 200         int                             num_bios_done;
 201         int                             send_complete;
 202         int                             num_errors;
 203         struct bintime                  ds_t0;
 204         devstat_tag_type                ds_tag_type;
 205         devstat_trans_flags             ds_trans_type;
 206         uint64_t                        io_len;
 207         uint64_t                        io_offset;
 208         struct ctl_be_block_softc       *softc;
 209         struct ctl_be_block_lun         *lun;
 210         STAILQ_ENTRY(ctl_be_block_io)   links;
 211 };
 212
 213 static int cbb_num_threads = 14;
 214 TUNABLE_INT("kern.cam.ctl.block.num_threads", &cbb_num_threads);
 215 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, block, CTLFLAG_RD, 0,
 216             "CAM Target Layer Block Backend");
 217 SYSCTL_INT(_kern_cam_ctl_block, OID_AUTO, num_threads, CTLFLAG_RW,
 218            &cbb_num_threads, 0, "Number of threads per backing file");
 219
 220 static struct ctl_be_block_io *ctl_alloc_beio(struct ctl_be_block_softc *softc);
 221 static void ctl_free_beio(struct ctl_be_block_io *beio);
 222 static int ctl_grow_beio(struct ctl_be_block_softc *softc, int count);
 223 #if 0
 224 static void ctl_shrink_beio(struct ctl_be_block_softc *softc);
 225 #endif
 226 static void ctl_complete_beio(struct ctl_be_block_io *beio);
 227 static int ctl_be_block_move_done(union ctl_io *io);
 228 static void ctl_be_block_biodone(struct bio *bio);
 229 static void ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 230                                     struct ctl_be_block_io *beio);
 231 static void ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 232                                        struct ctl_be_block_io *beio);
 233 static void ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 234                                    struct ctl_be_block_io *beio);
 235 static void ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
 236                                       struct ctl_be_block_io *beio);
 237 static void ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
 238                                     union ctl_io *io);
 239 static void ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 240                                   union ctl_io *io);
 241 static void ctl_be_block_worker(void *context, int pending);
 242 static int ctl_be_block_submit(union ctl_io *io);
 243 static int ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
 244                                    int flag, struct thread *td);
 245 static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
 246                                   struct ctl_lun_req *req);
 247 static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
 248                                  struct ctl_lun_req *req);
 249 static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
 250 static int ctl_be_block_open(struct ctl_be_block_softc *softc,
 251                              struct ctl_be_block_lun *be_lun,
 252                              struct ctl_lun_req *req);
 253 static int ctl_be_block_create(struct ctl_be_block_softc *softc,
 254                                struct ctl_lun_req *req);
 255 static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
 256                            struct ctl_lun_req *req);
 257 static int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
 258                                   struct ctl_lun_req *req);
 259 static int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
 260                                  struct ctl_lun_req *req);
 261 static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
 262                            struct ctl_lun_req *req);
 263 static void ctl_be_block_lun_shutdown(void *be_lun);
 264 static void ctl_be_block_lun_config_status(void *be_lun,
 265                                            ctl_lun_config_status status);
 266 static int ctl_be_block_config_write(union ctl_io *io);
 267 static int ctl_be_block_config_read(union ctl_io *io);
 268 static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb);
 269 int ctl_be_block_init(void);
 270
 271 static struct ctl_backend_driver ctl_be_block_driver =
 272 {
 273         .name = "block",
 274         .flags = CTL_BE_FLAG_HAS_CONFIG,
 275         .init = ctl_be_block_init,
 276         .data_submit = ctl_be_block_submit,
 277         .data_move_done = ctl_be_block_move_done,
 278         .config_read = ctl_be_block_config_read,
 279         .config_write = ctl_be_block_config_write,
 280         .ioctl = ctl_be_block_ioctl,
 281         .lun_info = ctl_be_block_lun_info
 282 };
 283
 284 MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend");
 285 CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
 286
 287 static struct ctl_be_block_io *
 288 ctl_alloc_beio(struct ctl_be_block_softc *softc)
 289 {
 290         struct ctl_be_block_io *beio;
 291         int count;
 292
 293         mtx_lock(&softc->lock);
 294
 295         beio = STAILQ_FIRST(&softc->beio_free_queue);
 296         if (beio != NULL) {
 297                 STAILQ_REMOVE(&softc->beio_free_queue, beio,
 298                               ctl_be_block_io, links);
 299         }
 300         mtx_unlock(&softc->lock);
 301
 302         if (beio != NULL) {
 303                 bzero(beio, sizeof(*beio));
 304                 beio->softc = softc;
 305                 return (beio);
 306         }
 307
 308         for (;;) {
 309
 310                 count = ctl_grow_beio(softc, /*count*/ 10);
 311
 312                 /*
 313                  * This shouldn't be possible, since ctl_grow_beio() uses a
 314                  * blocking malloc.
 315                  */
 316                 if (count == 0)
 317                         return (NULL);
 318
 319                 /*
 320                  * Since we have to drop the lock when we're allocating beio
 321                  * structures, it's possible someone else can come along and
 322                  * allocate the beio's we've just allocated.
 323                  */
 324                 mtx_lock(&softc->lock);
 325                 beio = STAILQ_FIRST(&softc->beio_free_queue);
 326                 if (beio != NULL) {
 327                         STAILQ_REMOVE(&softc->beio_free_queue, beio,
 328                                       ctl_be_block_io, links);
 329                 }
 330                 mtx_unlock(&softc->lock);
 331
 332                 if (beio != NULL) {
 333                         bzero(beio, sizeof(*beio));
 334                         beio->softc = softc;
 335                         break;
 336                 }
 337         }
 338         return (beio);
 339 }
 340
 341 static void
 342 ctl_free_beio(struct ctl_be_block_io *beio)
 343 {
 344         struct ctl_be_block_softc *softc;
 345         int duplicate_free;
 346         int i;
 347
 348         softc = beio->softc;
 349         duplicate_free = 0;
 350
 351         for (i = 0; i < beio->num_segs; i++) {
 352                 if (beio->sg_segs[i].addr == NULL)
 353                         duplicate_free++;
 354
 355                 uma_zfree(beio->lun->lun_zone, beio->sg_segs[i].addr);
 356                 beio->sg_segs[i].addr = NULL;
 357         }
 358
 359         if (duplicate_free > 0) {
 360                 printf("%s: %d duplicate frees out of %d segments\n", __func__,
 361                        duplicate_free, beio->num_segs);
 362         }
 363         mtx_lock(&softc->lock);
 364         STAILQ_INSERT_TAIL(&softc->beio_free_queue, beio, links);
 365         mtx_unlock(&softc->lock);
 366 }
 367
 368 static int
 369 ctl_grow_beio(struct ctl_be_block_softc *softc, int count)
 370 {
 371         int i;
 372
 373         for (i = 0; i < count; i++) {
 374                 struct ctl_be_block_io *beio;
 375
 376                 beio = (struct ctl_be_block_io *)malloc(sizeof(*beio),
 377                                                            M_CTLBLK,
 378                                                            M_WAITOK | M_ZERO);
 379                 beio->softc = softc;
 380                 mtx_lock(&softc->lock);
 381                 STAILQ_INSERT_TAIL(&softc->beio_free_queue, beio, links);
 382                 mtx_unlock(&softc->lock);
 383         }
 384
 385         return (i);
 386 }
 387
 388 #if 0
 389 static void
 390 ctl_shrink_beio(struct ctl_be_block_softc *softc)
 391 {
 392         struct ctl_be_block_io *beio, *beio_tmp;
 393
 394         mtx_lock(&softc->lock);
 395         STAILQ_FOREACH_SAFE(beio, &softc->beio_free_queue, links, beio_tmp) {
 396                 STAILQ_REMOVE(&softc->beio_free_queue, beio,
 397                               ctl_be_block_io, links);
 398                 free(beio, M_CTLBLK);
 399         }
 400         mtx_unlock(&softc->lock);
 401 }
 402 #endif
 403
 404 static void
 405 ctl_complete_beio(struct ctl_be_block_io *beio)
 406 {
 407         union ctl_io *io;
 408         int io_len;
 409
 410         io = beio->io;
 411
 412         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
 413                 io_len = beio->io_len;
 414         else
 415                 io_len = 0;
 416
 417         devstat_end_transaction(beio->lun->disk_stats,
 418                                 /*bytes*/ io_len,
 419                                 beio->ds_tag_type,
 420                                 beio->ds_trans_type,
 421                                 /*now*/ NULL,
 422                                 /*then*/&beio->ds_t0);
 423
 424         ctl_free_beio(beio);
 425         ctl_done(io);
 426 }
 427
 428 static int
 429 ctl_be_block_move_done(union ctl_io *io)
 430 {
 431         struct ctl_be_block_io *beio;
 432         struct ctl_be_block_lun *be_lun;
 433 #ifdef CTL_TIME_IO
 434         struct bintime cur_bt;
 435 #endif
 436
 437         beio = (struct ctl_be_block_io *)
 438                 io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr;
 439
 440         be_lun = beio->lun;
 441
 442         DPRINTF("entered\n");
 443
 444 #ifdef CTL_TIME_IO
 445         getbintime(&cur_bt);
 446         bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
 447         bintime_add(&io->io_hdr.dma_bt, &cur_bt);
 448         io->io_hdr.num_dmas++;
 449 #endif
 450
 451         /*
 452          * We set status at this point for read commands, and write
 453          * commands with errors.
 454          */
 455         if ((beio->bio_cmd == BIO_READ)
 456          && (io->io_hdr.port_status == 0)
 457          && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
 458          && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE))
 459                 ctl_set_success(&io->scsiio);
 460         else if ((io->io_hdr.port_status != 0)
 461               && ((io->io_hdr.flags & CTL_FLAG_ABORT) == 0)
 462               && ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE)) {
 463                 /*
 464                  * For hardware error sense keys, the sense key
 465                  * specific value is defined to be a retry count,
 466                  * but we use it to pass back an internal FETD
 467                  * error code.  XXX KDM  Hopefully the FETD is only
 468                  * using 16 bits for an error code, since that's
 469                  * all the space we have in the sks field.
 470                  */
 471                 ctl_set_internal_failure(&io->scsiio,
 472                                          /*sks_valid*/ 1,
 473                                          /*retry_count*/
 474                                          io->io_hdr.port_status);
 475         }
 476
 477         /*
 478          * If this is a read, or a write with errors, it is done.
 479          */
 480         if ((beio->bio_cmd == BIO_READ)
 481          || ((io->io_hdr.flags & CTL_FLAG_ABORT) != 0)
 482          || ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE)) {
 483                 ctl_complete_beio(beio);
 484                 return (0);
 485         }
 486
 487         /*
 488          * At this point, we have a write and the DMA completed
 489          * successfully.  We now have to queue it to the task queue to
 490          * execute the backend I/O.  That is because we do blocking
 491          * memory allocations, and in the file backing case, blocking I/O.
 492          * This move done routine is generally called in the SIM's
 493          * interrupt context, and therefore we cannot block.
 494          */
 495         mtx_lock(&be_lun->lock);
 496         /*
 497          * XXX KDM make sure that links is okay to use at this point.
 498          * Otherwise, we either need to add another field to ctl_io_hdr,
 499          * or deal with resource allocation here.
 500          */
 501         STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
 502         mtx_unlock(&be_lun->lock);
 503
 504         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
 505
 506         return (0);
 507 }
 508
 509 static void
 510 ctl_be_block_biodone(struct bio *bio)
 511 {
 512         struct ctl_be_block_io *beio;
 513         struct ctl_be_block_lun *be_lun;
 514         union ctl_io *io;
 515
 516         beio = bio->bio_caller1;
 517         be_lun = beio->lun;
 518         io = beio->io;
 519
 520         DPRINTF("entered\n");
 521
 522         mtx_lock(&be_lun->lock);
 523         if (bio->bio_error != 0)
 524                 beio->num_errors++;
 525
 526         beio->num_bios_done++;
 527
 528         /*
 529          * XXX KDM will this cause WITNESS to complain?  Holding a lock
 530          * during the free might cause it to complain.
 531          */
 532         g_destroy_bio(bio);
 533
 534         /*
 535          * If the send complete bit isn't set, or we aren't the last I/O to
 536          * complete, then we're done.
 537          */
 538         if ((beio->send_complete == 0)
 539          || (beio->num_bios_done < beio->num_bios_sent)) {
 540                 mtx_unlock(&be_lun->lock);
 541                 return;
 542         }
 543
 544         /*
 545          * At this point, we've verified that we are the last I/O to
 546          * complete, so it's safe to drop the lock.
 547          */
 548         mtx_unlock(&be_lun->lock);
 549
 550         /*
 551          * If there are any errors from the backing device, we fail the
 552          * entire I/O with a medium error.
 553          */
 554         if (beio->num_errors > 0) {
 555                 if (beio->bio_cmd == BIO_FLUSH) {
 556                         /* XXX KDM is there is a better error here? */
 557                         ctl_set_internal_failure(&io->scsiio,
 558                                                  /*sks_valid*/ 1,
 559                                                  /*retry_count*/ 0xbad2);
 560                 } else
 561                         ctl_set_medium_error(&io->scsiio);
 562                 ctl_complete_beio(beio);
 563                 return;
 564         }
 565
 566         /*
 567          * If this is a write or a flush, we're all done.
 568          * If this is a read, we can now send the data to the user.
 569          */
 570         if ((beio->bio_cmd == BIO_WRITE)
 571          || (beio->bio_cmd == BIO_FLUSH)) {
 572                 ctl_set_success(&io->scsiio);
 573                 ctl_complete_beio(beio);
 574         } else {
 575                 io->scsiio.be_move_done = ctl_be_block_move_done;
 576                 io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
 577                 io->scsiio.kern_data_len = beio->io_len;
 578                 io->scsiio.kern_total_len = beio->io_len;
 579                 io->scsiio.kern_rel_offset = 0;
 580                 io->scsiio.kern_data_resid = 0;
 581                 io->scsiio.kern_sg_entries = beio->num_segs;
 582                 io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
 583 #ifdef CTL_TIME_IO
 584                 getbintime(&io->io_hdr.dma_start_bt);
 585 #endif
 586                 ctl_datamove(io);
 587         }
 588 }
 589
 590 static void
 591 ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 592                         struct ctl_be_block_io *beio)
 593 {
 594         union ctl_io *io;
 595         struct mount *mountpoint;
 596         int error, lock_flags;
 597
 598         DPRINTF("entered\n");
 599
 600         io = beio->io;
 601
 602         (void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 603
 604         if (MNT_SHARED_WRITES(mountpoint)
 605          || ((mountpoint == NULL)
 606           && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 607                 lock_flags = LK_SHARED;
 608         else
 609                 lock_flags = LK_EXCLUSIVE;
 610
 611         vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 612
 613         binuptime(&beio->ds_t0);
 614         devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 615
 616         error = VOP_FSYNC(be_lun->vn, MNT_WAIT, curthread);
 617         VOP_UNLOCK(be_lun->vn, 0);
 618
 619         vn_finished_write(mountpoint);
 620
 621         if (error == 0)
 622                 ctl_set_success(&io->scsiio);
 623         else {
 624                 /* XXX KDM is there is a better error here? */
 625                 ctl_set_internal_failure(&io->scsiio,
 626                                          /*sks_valid*/ 1,
 627                                          /*retry_count*/ 0xbad1);
 628         }
 629
 630         ctl_complete_beio(beio);
 631 }
 632
 633 SDT_PROBE_DEFINE1(cbb, kernel, read, file_start, file_start, "uint64_t");
 634 SDT_PROBE_DEFINE1(cbb, kernel, write, file_start, file_start, "uint64_t");
 635 SDT_PROBE_DEFINE1(cbb, kernel, read, file_done, file_done,"uint64_t");
 636 SDT_PROBE_DEFINE1(cbb, kernel, write, file_done, file_done, "uint64_t");
 637
 638 static void
 639 ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 640                            struct ctl_be_block_io *beio)
 641 {
 642         struct ctl_be_block_filedata *file_data;
 643         union ctl_io *io;
 644         struct uio xuio;
 645         struct iovec *xiovec;
 646         int flags;
 647         int error, i;
 648
 649         DPRINTF("entered\n");
 650
 651         file_data = &be_lun->backend.file;
 652         io = beio->io;
 653         flags = beio->bio_flags;
 654
 655         if (beio->bio_cmd == BIO_READ) {
 656                 SDT_PROBE(cbb, kernel, read, file_start, 0, 0, 0, 0, 0);
 657         } else {
 658                 SDT_PROBE(cbb, kernel, write, file_start, 0, 0, 0, 0, 0);
 659         }
 660
 661         bzero(&xuio, sizeof(xuio));
 662         if (beio->bio_cmd == BIO_READ)
 663                 xuio.uio_rw = UIO_READ;
 664         else
 665                 xuio.uio_rw = UIO_WRITE;
 666
 667         xuio.uio_offset = beio->io_offset;
 668         xuio.uio_resid = beio->io_len;
 669         xuio.uio_segflg = UIO_SYSSPACE;
 670         xuio.uio_iov = beio->xiovecs;
 671         xuio.uio_iovcnt = beio->num_segs;
 672         xuio.uio_td = curthread;
 673
 674         for (i = 0, xiovec = xuio.uio_iov; i < xuio.uio_iovcnt; i++, xiovec++) {
 675                 xiovec->iov_base = beio->sg_segs[i].addr;
 676                 xiovec->iov_len = beio->sg_segs[i].len;
 677         }
 678
 679         if (beio->bio_cmd == BIO_READ) {
 680                 vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
 681
 682                 binuptime(&beio->ds_t0);
 683                 devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 684
 685                 /*
 686                  * UFS pays attention to IO_DIRECT for reads.  If the
 687                  * DIRECTIO option is configured into the kernel, it calls
 688                  * ffs_rawread().  But that only works for single-segment
 689                  * uios with user space addresses.  In our case, with a
 690                  * kernel uio, it still reads into the buffer cache, but it
 691                  * will just try to release the buffer from the cache later
 692                  * on in ffs_read().
 693                  *
 694                  * ZFS does not pay attention to IO_DIRECT for reads.
 695                  *
 696                  * UFS does not pay attention to IO_SYNC for reads.
 697                  *
 698                  * ZFS pays attention to IO_SYNC (which translates into the
 699                  * Solaris define FRSYNC for zfs_read()) for reads.  It
 700                  * attempts to sync the file before reading.
 701                  *
 702                  * So, to attempt to provide some barrier semantics in the
 703                  * BIO_ORDERED case, set both IO_DIRECT and IO_SYNC.
 704                  */
 705                 error = VOP_READ(be_lun->vn, &xuio, (flags & BIO_ORDERED) ?
 706                                  (IO_DIRECT|IO_SYNC) : 0, file_data->cred);
 707
 708                 VOP_UNLOCK(be_lun->vn, 0);
 709         } else {
 710                 struct mount *mountpoint;
 711                 int lock_flags;
 712
 713                 (void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 714
 715                 if (MNT_SHARED_WRITES(mountpoint)
 716                  || ((mountpoint == NULL)
 717                   && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 718                         lock_flags = LK_SHARED;
 719                 else
 720                         lock_flags = LK_EXCLUSIVE;
 721
 722                 vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 723
 724                 binuptime(&beio->ds_t0);
 725                 devstat_start_transaction(beio->lun->disk_stats, &beio->ds_t0);
 726
 727                 /*
 728                  * UFS pays attention to IO_DIRECT for writes.  The write
 729                  * is done asynchronously.  (Normally the write would just
 730                  * get put into cache.
 731                  *
 732                  * UFS pays attention to IO_SYNC for writes.  It will
 733                  * attempt to write the buffer out synchronously if that
 734                  * flag is set.
 735                  *
 736                  * ZFS does not pay attention to IO_DIRECT for writes.
 737                  *
 738                  * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC)
 739                  * for writes.  It will flush the transaction from the
 740                  * cache before returning.
 741                  *
 742                  * So if we've got the BIO_ORDERED flag set, we want
 743                  * IO_SYNC in either the UFS or ZFS case.
 744                  */
 745                 error = VOP_WRITE(be_lun->vn, &xuio, (flags & BIO_ORDERED) ?
 746                                   IO_SYNC : 0, file_data->cred);
 747                 VOP_UNLOCK(be_lun->vn, 0);
 748
 749                 vn_finished_write(mountpoint);
 750         }
 751
 752         /*
 753          * If we got an error, set the sense data to "MEDIUM ERROR" and
 754          * return the I/O to the user.
 755          */
 756         if (error != 0) {
 757                 char path_str[32];
 758
 759                 ctl_scsi_path_string(io, path_str, sizeof(path_str));
 760                 /*
 761                  * XXX KDM ZFS returns ENOSPC when the underlying
 762                  * filesystem fills up.  What kind of SCSI error should we
 763                  * return for that?
 764                  */
 765                 printf("%s%s command returned errno %d\n", path_str,
 766                        (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error);
 767                 ctl_set_medium_error(&io->scsiio);
 768                 ctl_complete_beio(beio);
 769                 return;
 770         }
 771
 772         /*
 773          * If this is a write, we're all done.
 774          * If this is a read, we can now send the data to the user.
 775          */
 776         if (beio->bio_cmd == BIO_WRITE) {
 777                 ctl_set_success(&io->scsiio);
 778                 SDT_PROBE(cbb, kernel, write, file_done, 0, 0, 0, 0, 0);
 779                 ctl_complete_beio(beio);
 780         } else {
 781                 SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
 782                 io->scsiio.be_move_done = ctl_be_block_move_done;
 783                 io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
 784                 io->scsiio.kern_data_len = beio->io_len;
 785                 io->scsiio.kern_total_len = beio->io_len;
 786                 io->scsiio.kern_rel_offset = 0;
 787                 io->scsiio.kern_data_resid = 0;
 788                 io->scsiio.kern_sg_entries = beio->num_segs;
 789                 io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
 790 #ifdef CTL_TIME_IO
 791                 getbintime(&io->io_hdr.dma_start_bt);
 792 #endif
 793                 ctl_datamove(io);
 794         }
 795 }
 796
 797 static void
 798 ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 799                        struct ctl_be_block_io *beio)
 800 {
 801         struct bio *bio;
 802         union ctl_io *io;
 803         struct ctl_be_block_devdata *dev_data;
 804
 805         dev_data = &be_lun->backend.dev;
 806         io = beio->io;
 807
 808         DPRINTF("entered\n");
 809
 810         /* This can't fail, it's a blocking allocation. */
 811         bio = g_alloc_bio();
 812
 813         bio->bio_cmd        = BIO_FLUSH;
 814         bio->bio_flags     |= BIO_ORDERED;
 815         bio->bio_dev        = dev_data->cdev;
 816         bio->bio_offset     = 0;
 817         bio->bio_data       = 0;
 818         bio->bio_done       = ctl_be_block_biodone;
 819         bio->bio_caller1    = beio;
 820         bio->bio_pblkno     = 0;
 821
 822         /*
 823          * We don't need to acquire the LUN lock here, because we are only
 824          * sending one bio, and so there is no other context to synchronize
 825          * with.
 826          */
 827         beio->num_bios_sent = 1;
 828         beio->send_complete = 1;
 829
 830         binuptime(&beio->ds_t0);
 831         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
 832
 833         (*dev_data->csw->d_strategy)(bio);
 834 }
 835
 836 static void
 837 ctl_be_block_dispatch_dev(struct ctl_be_block_lun *be_lun,
 838                           struct ctl_be_block_io *beio)
 839 {
 840         int i;
 841         struct bio *bio;
 842         struct ctl_be_block_devdata *dev_data;
 843         off_t cur_offset;
 844         int max_iosize;
 845
 846         DPRINTF("entered\n");
 847
 848         dev_data = &be_lun->backend.dev;
 849
 850         /*
 851          * We have to limit our I/O size to the maximum supported by the
 852          * backend device.  Hopefully it is MAXPHYS.  If the driver doesn't
 853          * set it properly, use DFLTPHYS.
 854          */
 855         max_iosize = dev_data->cdev->si_iosize_max;
 856         if (max_iosize < PAGE_SIZE)
 857                 max_iosize = DFLTPHYS;
 858
 859         cur_offset = beio->io_offset;
 860
 861         /*
 862          * XXX KDM need to accurately reflect the number of I/Os outstanding
 863          * to a device.
 864          */
 865         binuptime(&beio->ds_t0);
 866         devstat_start_transaction(be_lun->disk_stats, &beio->ds_t0);
 867
 868         for (i = 0; i < beio->num_segs; i++) {
 869                 size_t cur_size;
 870                 uint8_t *cur_ptr;
 871
 872                 cur_size = beio->sg_segs[i].len;
 873                 cur_ptr = beio->sg_segs[i].addr;
 874
 875                 while (cur_size > 0) {
 876                         /* This can't fail, it's a blocking allocation. */
 877                         bio = g_alloc_bio();
 878
 879                         KASSERT(bio != NULL, ("g_alloc_bio() failed!\n"));
 880
 881                         bio->bio_cmd = beio->bio_cmd;
 882                         bio->bio_flags |= beio->bio_flags;
 883                         bio->bio_dev = dev_data->cdev;
 884                         bio->bio_caller1 = beio;
 885                         bio->bio_length = min(cur_size, max_iosize);
 886                         bio->bio_offset = cur_offset;
 887                         bio->bio_data = cur_ptr;
 888                         bio->bio_done = ctl_be_block_biodone;
 889                         bio->bio_pblkno = cur_offset / be_lun->blocksize;
 890
 891                         cur_offset += bio->bio_length;
 892                         cur_ptr += bio->bio_length;
 893                         cur_size -= bio->bio_length;
 894
 895                         /*
 896                          * Make sure we set the complete bit just before we
 897                          * issue the last bio so we don't wind up with a
 898                          * race.
 899                          *
 900                          * Use the LUN mutex here instead of a combination
 901                          * of atomic variables for simplicity.
 902                          *
 903                          * XXX KDM we could have a per-IO lock, but that
 904                          * would cause additional per-IO setup and teardown
 905                          * overhead.  Hopefully there won't be too much
 906                          * contention on the LUN lock.
 907                          */
 908                         mtx_lock(&be_lun->lock);
 909
 910                         beio->num_bios_sent++;
 911
 912                         if ((i == beio->num_segs - 1)
 913                          && (cur_size == 0))
 914                                 beio->send_complete = 1;
 915
 916                         mtx_unlock(&be_lun->lock);
 917
 918                         (*dev_data->csw->d_strategy)(bio);
 919                 }
 920         }
 921 }
 922
 923 static void
 924 ctl_be_block_cw_dispatch(struct ctl_be_block_lun *be_lun,
 925                          union ctl_io *io)
 926 {
 927         struct ctl_be_block_io *beio;
 928         struct ctl_be_block_softc *softc;
 929
 930         DPRINTF("entered\n");
 931
 932         softc = be_lun->softc;
 933         beio = ctl_alloc_beio(softc);
 934         if (beio == NULL) {
 935                 /*
 936                  * This should not happen.  ctl_alloc_beio() will call
 937                  * ctl_grow_beio() with a blocking malloc as needed.
 938                  * A malloc with M_WAITOK should not fail.
 939                  */
 940                 ctl_set_busy(&io->scsiio);
 941                 ctl_done(io);
 942                 return;
 943         }
 944
 945         beio->io = io;
 946         beio->softc = softc;
 947         beio->lun = be_lun;
 948         io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr = beio;
 949
 950         switch (io->scsiio.cdb[0]) {
 951         case SYNCHRONIZE_CACHE:
 952         case SYNCHRONIZE_CACHE_16:
 953                 beio->bio_cmd = BIO_FLUSH;
 954                 beio->ds_trans_type = DEVSTAT_NO_DATA;
 955                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
 956                 beio->io_len = 0;
 957                 be_lun->lun_flush(be_lun, beio);
 958                 break;
 959         default:
 960                 panic("Unhandled CDB type %#x", io->scsiio.cdb[0]);
 961                 break;
 962         }
 963 }
 964
 965 SDT_PROBE_DEFINE1(cbb, kernel, read, start, start, "uint64_t");
 966 SDT_PROBE_DEFINE1(cbb, kernel, write, start, start, "uint64_t");
 967 SDT_PROBE_DEFINE1(cbb, kernel, read, alloc_done, alloc_done, "uint64_t");
 968 SDT_PROBE_DEFINE1(cbb, kernel, write, alloc_done, alloc_done, "uint64_t");
 969
 970 static void
 971 ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 972                            union ctl_io *io)
 973 {
 974         struct ctl_be_block_io *beio;
 975         struct ctl_be_block_softc *softc;
 976         struct ctl_lba_len lbalen;
 977         uint64_t len_left, io_size_bytes;
 978         int i;
 979
 980         softc = be_lun->softc;
 981
 982         DPRINTF("entered\n");
 983
 984         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
 985                 SDT_PROBE(cbb, kernel, read, start, 0, 0, 0, 0, 0);
 986         } else {
 987                 SDT_PROBE(cbb, kernel, write, start, 0, 0, 0, 0, 0);
 988         }
 989
 990         memcpy(&lbalen, io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
 991                sizeof(lbalen));
 992
 993         io_size_bytes = lbalen.len * be_lun->blocksize;
 994
 995         /*
 996          * XXX KDM this is temporary, until we implement chaining of beio
 997          * structures and multiple datamove calls to move all the data in
 998          * or out.
 999          */
1000         if (io_size_bytes > CTLBLK_MAX_IO_SIZE) {
1001                 printf("%s: IO length %ju > max io size %u\n", __func__,
1002                        io_size_bytes, CTLBLK_MAX_IO_SIZE);
1003                 ctl_set_invalid_field(&io->scsiio,
1004                                       /*sks_valid*/ 0,
1005                                       /*command*/ 1,
1006                                       /*field*/ 0,
1007                                       /*bit_valid*/ 0,
1008                                       /*bit*/ 0);
1009                 ctl_done(io);
1010                 return;
1011         }
1012
1013         beio = ctl_alloc_beio(softc);
1014         if (beio == NULL) {
1015                 /*
1016                  * This should not happen.  ctl_alloc_beio() will call
1017                  * ctl_grow_beio() with a blocking malloc as needed.
1018                  * A malloc with M_WAITOK should not fail.
1019                  */
1020                 ctl_set_busy(&io->scsiio);
1021                 ctl_done(io);
1022                 return;
1023         }
1024
1025         beio->io = io;
1026         beio->softc = softc;
1027         beio->lun = be_lun;
1028         io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr = beio;
1029
1030         /*
1031          * If the I/O came down with an ordered or head of queue tag, set
1032          * the BIO_ORDERED attribute.  For head of queue tags, that's
1033          * pretty much the best we can do.
1034          *
1035          * XXX KDM we don't have a great way to easily know about the FUA
1036          * bit right now (it is decoded in ctl_read_write(), but we don't
1037          * pass that knowledge to the backend), and in any case we would
1038          * need to determine how to handle it.
1039          */
1040         if ((io->scsiio.tag_type == CTL_TAG_ORDERED)
1041          || (io->scsiio.tag_type == CTL_TAG_HEAD_OF_QUEUE))
1042                 beio->bio_flags = BIO_ORDERED;
1043
1044         switch (io->scsiio.tag_type) {
1045         case CTL_TAG_ORDERED:
1046                 beio->ds_tag_type = DEVSTAT_TAG_ORDERED;
1047                 break;
1048         case CTL_TAG_HEAD_OF_QUEUE:
1049                 beio->ds_tag_type = DEVSTAT_TAG_HEAD;
1050                 break;
1051         case CTL_TAG_UNTAGGED:
1052         case CTL_TAG_SIMPLE:
1053         case CTL_TAG_ACA:
1054         default:
1055                 beio->ds_tag_type = DEVSTAT_TAG_SIMPLE;
1056                 break;
1057         }
1058
1059         /*
1060          * This path handles read and write only.  The config write path
1061          * handles flush operations.
1062          */
1063         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
1064                 beio->bio_cmd = BIO_READ;
1065                 beio->ds_trans_type = DEVSTAT_READ;
1066         } else {
1067                 beio->bio_cmd = BIO_WRITE;
1068                 beio->ds_trans_type = DEVSTAT_WRITE;
1069         }
1070
1071         beio->io_len = lbalen.len * be_lun->blocksize;
1072         beio->io_offset = lbalen.lba * be_lun->blocksize;
1073
1074         DPRINTF("%s at LBA %jx len %u\n",
1075                (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE",
1076                (uintmax_t)lbalen.lba, lbalen.len);
1077
1078         for (i = 0, len_left = io_size_bytes; i < CTLBLK_MAX_SEGS &&
1079              len_left > 0; i++) {
1080
1081                 /*
1082                  * Setup the S/G entry for this chunk.
1083                  */
1084                 beio->sg_segs[i].len = min(MAXPHYS, len_left);
1085                 beio->sg_segs[i].addr = uma_zalloc(be_lun->lun_zone, M_WAITOK);
1086
1087                 DPRINTF("segment %d addr %p len %zd\n", i,
1088                         beio->sg_segs[i].addr, beio->sg_segs[i].len);
1089
1090                 beio->num_segs++;
1091                 len_left -= beio->sg_segs[i].len;
1092         }
1093
1094         /*
1095          * For the read case, we need to read the data into our buffers and
1096          * then we can send it back to the user.  For the write case, we
1097          * need to get the data from the user first.
1098          */
1099         if (beio->bio_cmd == BIO_READ) {
1100                 SDT_PROBE(cbb, kernel, read, alloc_done, 0, 0, 0, 0, 0);
1101                 be_lun->dispatch(be_lun, beio);
1102         } else {
1103                 SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0);
1104                 io->scsiio.be_move_done = ctl_be_block_move_done;
1105                 io->scsiio.kern_data_ptr = (uint8_t *)beio->sg_segs;
1106                 io->scsiio.kern_data_len = beio->io_len;
1107                 io->scsiio.kern_total_len = beio->io_len;
1108                 io->scsiio.kern_rel_offset = 0;
1109                 io->scsiio.kern_data_resid = 0;
1110                 io->scsiio.kern_sg_entries = beio->num_segs;
1111                 io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
1112 #ifdef CTL_TIME_IO
1113                 getbintime(&io->io_hdr.dma_start_bt);
1114 #endif
1115                 ctl_datamove(io);
1116         }
1117 }
1118
1119 static void
1120 ctl_be_block_worker(void *context, int pending)
1121 {
1122         struct ctl_be_block_lun *be_lun;
1123         struct ctl_be_block_softc *softc;
1124         union ctl_io *io;
1125
1126         be_lun = (struct ctl_be_block_lun *)context;
1127         softc = be_lun->softc;
1128
1129         DPRINTF("entered\n");
1130
1131         mtx_lock(&be_lun->lock);
1132         for (;;) {
1133                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->datamove_queue);
1134                 if (io != NULL) {
1135                         struct ctl_be_block_io *beio;
1136
1137                         DPRINTF("datamove queue\n");
1138
1139                         STAILQ_REMOVE(&be_lun->datamove_queue, &io->io_hdr,
1140                                       ctl_io_hdr, links);
1141
1142                         mtx_unlock(&be_lun->lock);
1143
1144                         beio = (struct ctl_be_block_io *)
1145                             io->io_hdr.ctl_private[CTL_PRIV_BACKEND].ptr;
1146
1147                         be_lun->dispatch(be_lun, beio);
1148
1149                         mtx_lock(&be_lun->lock);
1150                         continue;
1151                 }
1152                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->config_write_queue);
1153                 if (io != NULL) {
1154
1155                         DPRINTF("config write queue\n");
1156
1157                         STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr,
1158                                       ctl_io_hdr, links);
1159
1160                         mtx_unlock(&be_lun->lock);
1161
1162                         ctl_be_block_cw_dispatch(be_lun, io);
1163
1164                         mtx_lock(&be_lun->lock);
1165                         continue;
1166                 }
1167                 io = (union ctl_io *)STAILQ_FIRST(&be_lun->input_queue);
1168                 if (io != NULL) {
1169                         DPRINTF("input queue\n");
1170
1171                         STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr,
1172                                       ctl_io_hdr, links);
1173                         mtx_unlock(&be_lun->lock);
1174
1175                         /*
1176                          * We must drop the lock, since this routine and
1177                          * its children may sleep.
1178                          */
1179                         ctl_be_block_dispatch(be_lun, io);
1180
1181                         mtx_lock(&be_lun->lock);
1182                         continue;
1183                 }
1184
1185                 /*
1186                  * If we get here, there is no work left in the queues, so
1187                  * just break out and let the task queue go to sleep.
1188                  */
1189                 break;
1190         }
1191         mtx_unlock(&be_lun->lock);
1192 }
1193
1194 /*
1195  * Entry point from CTL to the backend for I/O.  We queue everything to a
1196  * work thread, so this just puts the I/O on a queue and wakes up the
1197  * thread.
1198  */
1199 static int
1200 ctl_be_block_submit(union ctl_io *io)
1201 {
1202         struct ctl_be_block_lun *be_lun;
1203         struct ctl_be_lun *ctl_be_lun;
1204         int retval;
1205
1206         DPRINTF("entered\n");
1207
1208         retval = CTL_RETVAL_COMPLETE;
1209
1210         ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
1211                 CTL_PRIV_BACKEND_LUN].ptr;
1212         be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun;
1213
1214         /*
1215          * Make sure we only get SCSI I/O.
1216          */
1217         KASSERT(io->io_hdr.io_type == CTL_IO_SCSI, ("Non-SCSI I/O (type "
1218                 "%#x) encountered", io->io_hdr.io_type));
1219
1220         mtx_lock(&be_lun->lock);
1221         /*
1222          * XXX KDM make sure that links is okay to use at this point.
1223          * Otherwise, we either need to add another field to ctl_io_hdr,
1224          * or deal with resource allocation here.
1225          */
1226         STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
1227         mtx_unlock(&be_lun->lock);
1228
1229         taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
1230
1231         return (retval);
1232 }
1233
1234 static int
1235 ctl_be_block_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
1236                         int flag, struct thread *td)
1237 {
1238         struct ctl_be_block_softc *softc;
1239         int error;
1240
1241         softc = &backend_block_softc;
1242
1243         error = 0;
1244
1245         switch (cmd) {
1246         case CTL_LUN_REQ: {
1247                 struct ctl_lun_req *lun_req;
1248
1249                 lun_req = (struct ctl_lun_req *)addr;
1250
1251                 switch (lun_req->reqtype) {
1252                 case CTL_LUNREQ_CREATE:
1253                         error = ctl_be_block_create(softc, lun_req);
1254                         break;
1255                 case CTL_LUNREQ_RM:
1256                         error = ctl_be_block_rm(softc, lun_req);
1257                         break;
1258                 case CTL_LUNREQ_MODIFY:
1259                         error = ctl_be_block_modify(softc, lun_req);
1260                         break;
1261                 default:
1262                         lun_req->status = CTL_LUN_ERROR;
1263                         snprintf(lun_req->error_str, sizeof(lun_req->error_str),
1264                                  "%s: invalid LUN request type %d", __func__,
1265                                  lun_req->reqtype);
1266                         break;
1267                 }
1268                 break;
1269         }
1270         default:
1271                 error = ENOTTY;
1272                 break;
1273         }
1274
1275         return (error);
1276 }
1277
1278 static int
1279 ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1280 {
1281         struct ctl_be_block_filedata *file_data;
1282         struct ctl_lun_create_params *params;
1283         struct vattr                  vattr;
1284         int                           error;
1285
1286         error = 0;
1287         file_data = &be_lun->backend.file;
1288         params = &req->reqdata.create;
1289
1290         be_lun->dev_type = CTL_BE_BLOCK_FILE;
1291         be_lun->dispatch = ctl_be_block_dispatch_file;
1292         be_lun->lun_flush = ctl_be_block_flush_file;
1293
1294         error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
1295         if (error != 0) {
1296                 snprintf(req->error_str, sizeof(req->error_str),
1297                          "error calling VOP_GETATTR() for file %s",
1298                          be_lun->dev_path);
1299                 return (error);
1300         }
1301
1302         /*
1303          * Verify that we have the ability to upgrade to exclusive
1304          * access on this file so we can trap errors at open instead
1305          * of reporting them during first access.
1306          */
1307         if (VOP_ISLOCKED(be_lun->vn) != LK_EXCLUSIVE) {
1308                 vn_lock(be_lun->vn, LK_UPGRADE | LK_RETRY);
1309                 if (be_lun->vn->v_iflag & VI_DOOMED) {
1310                         error = EBADF;
1311                         snprintf(req->error_str, sizeof(req->error_str),
1312                                  "error locking file %s", be_lun->dev_path);
1313                         return (error);
1314                 }
1315         }
1316
1317
1318         file_data->cred = crhold(curthread->td_ucred);
1319         if (params->lun_size_bytes != 0)
1320                 be_lun->size_bytes = params->lun_size_bytes;
1321         else
1322                 be_lun->size_bytes = vattr.va_size;
1323         /*
1324          * We set the multi thread flag for file operations because all
1325          * filesystems (in theory) are capable of allowing multiple readers
1326          * of a file at once.  So we want to get the maximum possible
1327          * concurrency.
1328          */
1329         be_lun->flags |= CTL_BE_BLOCK_LUN_MULTI_THREAD;
1330
1331         /*
1332          * XXX KDM vattr.va_blocksize may be larger than 512 bytes here.
1333          * With ZFS, it is 131072 bytes.  Block sizes that large don't work
1334          * with disklabel and UFS on FreeBSD at least.  Large block sizes
1335          * may not work with other OSes as well.  So just export a sector
1336          * size of 512 bytes, which should work with any OS or
1337          * application.  Since our backing is a file, any block size will
1338          * work fine for the backing store.
1339          */
1340 #if 0
1341         be_lun->blocksize= vattr.va_blocksize;
1342 #endif
1343         if (params->blocksize_bytes != 0)
1344                 be_lun->blocksize = params->blocksize_bytes;
1345         else
1346                 be_lun->blocksize = 512;
1347
1348         /*
1349          * Sanity check.  The media size has to be at least one
1350          * sector long.
1351          */
1352         if (be_lun->size_bytes < be_lun->blocksize) {
1353                 error = EINVAL;
1354                 snprintf(req->error_str, sizeof(req->error_str),
1355                          "file %s size %ju < block size %u", be_lun->dev_path,
1356                          (uintmax_t)be_lun->size_bytes, be_lun->blocksize);
1357         }
1358         return (error);
1359 }
1360
1361 static int
1362 ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1363 {
1364         struct ctl_lun_create_params *params;
1365         struct vattr                  vattr;
1366         struct cdev                  *dev;
1367         struct cdevsw                *devsw;
1368         int                           error;
1369
1370         params = &req->reqdata.create;
1371
1372         be_lun->dev_type = CTL_BE_BLOCK_DEV;
1373         be_lun->dispatch = ctl_be_block_dispatch_dev;
1374         be_lun->lun_flush = ctl_be_block_flush_dev;
1375         be_lun->backend.dev.cdev = be_lun->vn->v_rdev;
1376         be_lun->backend.dev.csw = dev_refthread(be_lun->backend.dev.cdev,
1377                                              &be_lun->backend.dev.dev_ref);
1378         if (be_lun->backend.dev.csw == NULL)
1379                 panic("Unable to retrieve device switch");
1380
1381         error = VOP_GETATTR(be_lun->vn, &vattr, NOCRED);
1382         if (error) {
1383                 snprintf(req->error_str, sizeof(req->error_str),
1384                          "%s: error getting vnode attributes for device %s",
1385                          __func__, be_lun->dev_path);
1386                 return (error);
1387         }
1388
1389         dev = be_lun->vn->v_rdev;
1390         devsw = dev->si_devsw;
1391         if (!devsw->d_ioctl) {
1392                 snprintf(req->error_str, sizeof(req->error_str),
1393                          "%s: no d_ioctl for device %s!", __func__,
1394                          be_lun->dev_path);
1395                 return (ENODEV);
1396         }
1397
1398         error = devsw->d_ioctl(dev, DIOCGSECTORSIZE,
1399                                (caddr_t)&be_lun->blocksize, FREAD,
1400                                curthread);
1401         if (error) {
1402                 snprintf(req->error_str, sizeof(req->error_str),
1403                          "%s: error %d returned for DIOCGSECTORSIZE ioctl "
1404                          "on %s!", __func__, error, be_lun->dev_path);
1405                 return (error);
1406         }
1407
1408         /*
1409          * If the user has asked for a blocksize that is greater than the
1410          * backing device's blocksize, we can do it only if the blocksize
1411          * the user is asking for is an even multiple of the underlying
1412          * device's blocksize.
1413          */
1414         if ((params->blocksize_bytes != 0)
1415          && (params->blocksize_bytes > be_lun->blocksize)) {
1416                 uint32_t bs_multiple, tmp_blocksize;
1417
1418                 bs_multiple = params->blocksize_bytes / be_lun->blocksize;
1419
1420                 tmp_blocksize = bs_multiple * be_lun->blocksize;
1421
1422                 if (tmp_blocksize == params->blocksize_bytes) {
1423                         be_lun->blocksize = params->blocksize_bytes;
1424                 } else {
1425                         snprintf(req->error_str, sizeof(req->error_str),
1426                                  "%s: requested blocksize %u is not an even "
1427                                  "multiple of backing device blocksize %u",
1428                                  __func__, params->blocksize_bytes,
1429                                  be_lun->blocksize);
1430                         return (EINVAL);
1431
1432                 }
1433         } else if ((params->blocksize_bytes != 0)
1434                 && (params->blocksize_bytes != be_lun->blocksize)) {
1435                 snprintf(req->error_str, sizeof(req->error_str),
1436                          "%s: requested blocksize %u < backing device "
1437                          "blocksize %u", __func__, params->blocksize_bytes,
1438                          be_lun->blocksize);
1439                 return (EINVAL);
1440         }
1441
1442         error = devsw->d_ioctl(dev, DIOCGMEDIASIZE,
1443                                (caddr_t)&be_lun->size_bytes, FREAD,
1444                                curthread);
1445         if (error) {
1446                 snprintf(req->error_str, sizeof(req->error_str),
1447                          "%s: error %d returned for DIOCGMEDIASIZE "
1448                          " ioctl on %s!", __func__, error,
1449                          be_lun->dev_path);
1450                 return (error);
1451         }
1452
1453         if (params->lun_size_bytes != 0) {
1454                 if (params->lun_size_bytes > be_lun->size_bytes) {
1455                         snprintf(req->error_str, sizeof(req->error_str),
1456                                  "%s: requested LUN size %ju > backing device "
1457                                  "size %ju", __func__,
1458                                  (uintmax_t)params->lun_size_bytes,
1459                                  (uintmax_t)be_lun->size_bytes);
1460                         return (EINVAL);
1461                 }
1462
1463                 be_lun->size_bytes = params->lun_size_bytes;
1464         }
1465
1466         return (0);
1467 }
1468
1469 static int
1470 ctl_be_block_close(struct ctl_be_block_lun *be_lun)
1471 {
1472         DROP_GIANT();
1473         if (be_lun->vn) {
1474                 int flags = FREAD | FWRITE;
1475
1476                 switch (be_lun->dev_type) {
1477                 case CTL_BE_BLOCK_DEV:
1478                         if (be_lun->backend.dev.csw) {
1479                                 dev_relthread(be_lun->backend.dev.cdev,
1480                                               be_lun->backend.dev.dev_ref);
1481                                 be_lun->backend.dev.csw  = NULL;
1482                                 be_lun->backend.dev.cdev = NULL;
1483                         }
1484                         break;
1485                 case CTL_BE_BLOCK_FILE:
1486                         break;
1487                 case CTL_BE_BLOCK_NONE:
1488                         break;
1489                 default:
1490                         panic("Unexpected backend type.");
1491                         break;
1492                 }
1493
1494                 (void)vn_close(be_lun->vn, flags, NOCRED, curthread);
1495                 be_lun->vn = NULL;
1496
1497                 switch (be_lun->dev_type) {
1498                 case CTL_BE_BLOCK_DEV:
1499                         break;
1500                 case CTL_BE_BLOCK_FILE:
1501                         if (be_lun->backend.file.cred != NULL) {
1502                                 crfree(be_lun->backend.file.cred);
1503                                 be_lun->backend.file.cred = NULL;
1504                         }
1505                         break;
1506                 case CTL_BE_BLOCK_NONE:
1507                         break;
1508                 default:
1509                         panic("Unexpected backend type.");
1510                         break;
1511                 }
1512         }
1513         PICKUP_GIANT();
1514
1515         return (0);
1516 }
1517
1518 static int
1519 ctl_be_block_open(struct ctl_be_block_softc *softc,
1520                        struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
1521 {
1522         struct nameidata nd;
1523         int              flags;
1524         int              error;
1525
1526         /*
1527          * XXX KDM allow a read-only option?
1528          */
1529         flags = FREAD | FWRITE;
1530         error = 0;
1531
1532         if (rootvnode == NULL) {
1533                 snprintf(req->error_str, sizeof(req->error_str),
1534                          "%s: Root filesystem is not mounted", __func__);
1535                 return (1);
1536         }
1537
1538         if (!curthread->td_proc->p_fd->fd_cdir) {
1539                 curthread->td_proc->p_fd->fd_cdir = rootvnode;
1540                 VREF(rootvnode);
1541         }
1542         if (!curthread->td_proc->p_fd->fd_rdir) {
1543                 curthread->td_proc->p_fd->fd_rdir = rootvnode;
1544                 VREF(rootvnode);
1545         }
1546         if (!curthread->td_proc->p_fd->fd_jdir) {
1547                 curthread->td_proc->p_fd->fd_jdir = rootvnode;
1548                 VREF(rootvnode);
1549         }
1550
1551  again:
1552         NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, be_lun->dev_path, curthread);
1553         error = vn_open(&nd, &flags, 0, NULL);
1554         if (error) {
1555                 /*
1556                  * This is the only reasonable guess we can make as far as
1557                  * path if the user doesn't give us a fully qualified path.
1558                  * If they want to specify a file, they need to specify the
1559                  * full path.
1560                  */
1561                 if (be_lun->dev_path[0] != '/') {
1562                         char *dev_path = "/dev/";
1563                         char *dev_name;
1564
1565                         /* Try adding device path at beginning of name */
1566                         dev_name = malloc(strlen(be_lun->dev_path)
1567                                         + strlen(dev_path) + 1,
1568                                           M_CTLBLK, M_WAITOK);
1569                         if (dev_name) {
1570                                 sprintf(dev_name, "%s%s", dev_path,
1571                                         be_lun->dev_path);
1572                                 free(be_lun->dev_path, M_CTLBLK);
1573                                 be_lun->dev_path = dev_name;
1574                                 goto again;
1575                         }
1576                 }
1577                 snprintf(req->error_str, sizeof(req->error_str),
1578                          "%s: error opening %s", __func__, be_lun->dev_path);
1579                 return (error);
1580         }
1581
1582         NDFREE(&nd, NDF_ONLY_PNBUF);
1583
1584         be_lun->vn = nd.ni_vp;
1585
1586         /* We only support disks and files. */
1587         if (vn_isdisk(be_lun->vn, &error)) {
1588                 error = ctl_be_block_open_dev(be_lun, req);
1589         } else if (be_lun->vn->v_type == VREG) {
1590                 error = ctl_be_block_open_file(be_lun, req);
1591         } else {
1592                 error = EINVAL;
1593                 snprintf(req->error_str, sizeof(req->error_str),
1594                          "%s is not a disk or plain file", be_lun->dev_path);
1595         }
1596         VOP_UNLOCK(be_lun->vn, 0);
1597
1598         if (error != 0) {
1599                 ctl_be_block_close(be_lun);
1600                 return (error);
1601         }
1602
1603         be_lun->blocksize_shift = fls(be_lun->blocksize) - 1;
1604         be_lun->size_blocks = be_lun->size_bytes >> be_lun->blocksize_shift;
1605
1606         return (0);
1607 }
1608
1609 static int
1610 ctl_be_block_mem_ctor(void *mem, int size, void *arg, int flags)
1611 {
1612         return (0);
1613 }
1614
1615 static void
1616 ctl_be_block_mem_dtor(void *mem, int size, void *arg)
1617 {
1618         bzero(mem, size);
1619 }
1620
1621 static int
1622 ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
1623 {
1624         struct ctl_be_block_lun *be_lun;
1625         struct ctl_lun_create_params *params;
1626         struct ctl_be_arg *file_arg;
1627         char tmpstr[32];
1628         int retval, num_threads;
1629         int i;
1630
1631         params = &req->reqdata.create;
1632         retval = 0;
1633
1634         num_threads = cbb_num_threads;
1635
1636         file_arg = NULL;
1637
1638         be_lun = malloc(sizeof(*be_lun), M_CTLBLK, M_ZERO | M_WAITOK);
1639
1640         be_lun->softc = softc;
1641         STAILQ_INIT(&be_lun->input_queue);
1642         STAILQ_INIT(&be_lun->config_write_queue);
1643         STAILQ_INIT(&be_lun->datamove_queue);
1644         STAILQ_INIT(&be_lun->ctl_be_lun.options);
1645         sprintf(be_lun->lunname, "cblk%d", softc->num_luns);
1646         mtx_init(&be_lun->lock, be_lun->lunname, NULL, MTX_DEF);
1647
1648         be_lun->lun_zone = uma_zcreate(be_lun->lunname, MAXPHYS,
1649             ctl_be_block_mem_ctor, ctl_be_block_mem_dtor, NULL, NULL,
1650             /*align*/ 0, /*flags*/0);
1651
1652         if (be_lun->lun_zone == NULL) {
1653                 snprintf(req->error_str, sizeof(req->error_str),
1654                          "%s: error allocating UMA zone", __func__);
1655                 goto bailout_error;
1656         }
1657
1658         if (params->flags & CTL_LUN_FLAG_DEV_TYPE)
1659                 be_lun->ctl_be_lun.lun_type = params->device_type;
1660         else
1661                 be_lun->ctl_be_lun.lun_type = T_DIRECT;
1662
1663         if (be_lun->ctl_be_lun.lun_type == T_DIRECT) {
1664                 for (i = 0; i < req->num_be_args; i++) {
1665                         if (strcmp(req->kern_be_args[i].kname, "file") == 0) {
1666                                 file_arg = &req->kern_be_args[i];
1667                                 break;
1668                         }
1669                 }
1670
1671                 if (file_arg == NULL) {
1672                         snprintf(req->error_str, sizeof(req->error_str),
1673                                  "%s: no file argument specified", __func__);
1674                         goto bailout_error;
1675                 }
1676
1677                 be_lun->dev_path = malloc(file_arg->vallen, M_CTLBLK,
1678                                           M_WAITOK | M_ZERO);
1679
1680                 strlcpy(be_lun->dev_path, (char *)file_arg->kvalue,
1681                         file_arg->vallen);
1682
1683                 retval = ctl_be_block_open(softc, be_lun, req);
1684                 if (retval != 0) {
1685                         retval = 0;
1686                         goto bailout_error;
1687                 }
1688
1689                 /*
1690                  * Tell the user the size of the file/device.
1691                  */
1692                 params->lun_size_bytes = be_lun->size_bytes;
1693
1694                 /*
1695                  * The maximum LBA is the size - 1.
1696                  */
1697                 be_lun->ctl_be_lun.maxlba = be_lun->size_blocks - 1;
1698         } else {
1699                 /*
1700                  * For processor devices, we don't have any size.
1701                  */
1702                 be_lun->blocksize = 0;
1703                 be_lun->size_blocks = 0;
1704                 be_lun->size_bytes = 0;
1705                 be_lun->ctl_be_lun.maxlba = 0;
1706                 params->lun_size_bytes = 0;
1707
1708                 /*
1709                  * Default to just 1 thread for processor devices.
1710                  */
1711                 num_threads = 1;
1712         }
1713
1714         /*
1715          * XXX This searching loop might be refactored to be combined with
1716          * the loop above,
1717          */
1718         for (i = 0; i < req->num_be_args; i++) {
1719                 if (strcmp(req->kern_be_args[i].kname, "num_threads") == 0) {
1720                         struct ctl_be_arg *thread_arg;
1721                         char num_thread_str[16];
1722                         int tmp_num_threads;
1723
1724
1725                         thread_arg = &req->kern_be_args[i];
1726
1727                         strlcpy(num_thread_str, (char *)thread_arg->kvalue,
1728                                 min(thread_arg->vallen,
1729                                 sizeof(num_thread_str)));
1730
1731                         tmp_num_threads = strtol(num_thread_str, NULL, 0);
1732
1733                         /*
1734                          * We don't let the user specify less than one
1735                          * thread, but hope he's clueful enough not to
1736                          * specify 1000 threads.
1737                          */
1738                         if (tmp_num_threads < 1) {
1739                                 snprintf(req->error_str, sizeof(req->error_str),
1740                                          "%s: invalid number of threads %s",
1741                                          __func__, num_thread_str);
1742                                 goto bailout_error;
1743                         }
1744
1745                         num_threads = tmp_num_threads;
1746                 } else if (strcmp(req->kern_be_args[i].kname, "file") != 0 &&
1747                     strcmp(req->kern_be_args[i].kname, "dev") != 0) {
1748                         struct ctl_be_lun_option *opt;
1749
1750                         opt = malloc(sizeof(*opt), M_CTLBLK, M_WAITOK);
1751                         opt->name = malloc(strlen(req->kern_be_args[i].kname) + 1, M_CTLBLK, M_WAITOK);
1752                         strcpy(opt->name, req->kern_be_args[i].kname);
1753                         opt->value = malloc(strlen(req->kern_be_args[i].kvalue) + 1, M_CTLBLK, M_WAITOK);
1754                         strcpy(opt->value, req->kern_be_args[i].kvalue);
1755                         STAILQ_INSERT_TAIL(&be_lun->ctl_be_lun.options, opt, links);
1756                 }
1757         }
1758
1759         be_lun->flags = CTL_BE_BLOCK_LUN_UNCONFIGURED;
1760         be_lun->ctl_be_lun.flags = CTL_LUN_FLAG_PRIMARY;
1761         be_lun->ctl_be_lun.be_lun = be_lun;
1762         be_lun->ctl_be_lun.blocksize = be_lun->blocksize;
1763         /* Tell the user the blocksize we ended up using */
1764         params->blocksize_bytes = be_lun->blocksize;
1765         if (params->flags & CTL_LUN_FLAG_ID_REQ) {
1766                 be_lun->ctl_be_lun.req_lun_id = params->req_lun_id;
1767                 be_lun->ctl_be_lun.flags |= CTL_LUN_FLAG_ID_REQ;
1768         } else
1769                 be_lun->ctl_be_lun.req_lun_id = 0;
1770
1771         be_lun->ctl_be_lun.lun_shutdown = ctl_be_block_lun_shutdown;
1772         be_lun->ctl_be_lun.lun_config_status =
1773                 ctl_be_block_lun_config_status;
1774         be_lun->ctl_be_lun.be = &ctl_be_block_driver;
1775
1776         if ((params->flags & CTL_LUN_FLAG_SERIAL_NUM) == 0) {
1777                 snprintf(tmpstr, sizeof(tmpstr), "MYSERIAL%4d",
1778                          softc->num_luns);
1779                 strncpy((char *)be_lun->ctl_be_lun.serial_num, tmpstr,
1780                         ctl_min(sizeof(be_lun->ctl_be_lun.serial_num),
1781                         sizeof(tmpstr)));
1782
1783                 /* Tell the user what we used for a serial number */
1784                 strncpy((char *)params->serial_num, tmpstr,
1785                         ctl_min(sizeof(params->serial_num), sizeof(tmpstr)));
1786         } else {
1787                 strncpy((char *)be_lun->ctl_be_lun.serial_num,
1788                         params->serial_num,
1789                         ctl_min(sizeof(be_lun->ctl_be_lun.serial_num),
1790                         sizeof(params->serial_num)));
1791         }
1792         if ((params->flags & CTL_LUN_FLAG_DEVID) == 0) {
1793                 snprintf(tmpstr, sizeof(tmpstr), "MYDEVID%4d", softc->num_luns);
1794                 strncpy((char *)be_lun->ctl_be_lun.device_id, tmpstr,
1795                         ctl_min(sizeof(be_lun->ctl_be_lun.device_id),
1796                         sizeof(tmpstr)));
1797
1798                 /* Tell the user what we used for a device ID */
1799                 strncpy((char *)params->device_id, tmpstr,
1800                         ctl_min(sizeof(params->device_id), sizeof(tmpstr)));
1801         } else {
1802                 strncpy((char *)be_lun->ctl_be_lun.device_id,
1803                         params->device_id,
1804                         ctl_min(sizeof(be_lun->ctl_be_lun.device_id),
1805                                 sizeof(params->device_id)));
1806         }
1807
1808         TASK_INIT(&be_lun->io_task, /*priority*/0, ctl_be_block_worker, be_lun);
1809
1810         be_lun->io_taskqueue = taskqueue_create(be_lun->lunname, M_WAITOK,
1811             taskqueue_thread_enqueue, /*context*/&be_lun->io_taskqueue);
1812
1813         if (be_lun->io_taskqueue == NULL) {
1814                 snprintf(req->error_str, sizeof(req->error_str),
1815                          "%s: Unable to create taskqueue", __func__);
1816                 goto bailout_error;
1817         }
1818
1819         /*
1820          * Note that we start the same number of threads by default for
1821          * both the file case and the block device case.  For the file
1822          * case, we need multiple threads to allow concurrency, because the
1823          * vnode interface is designed to be a blocking interface.  For the
1824          * block device case, ZFS zvols at least will block the caller's
1825          * context in many instances, and so we need multiple threads to
1826          * overcome that problem.  Other block devices don't need as many
1827          * threads, but they shouldn't cause too many problems.
1828          *
1829          * If the user wants to just have a single thread for a block
1830          * device, he can specify that when the LUN is created, or change
1831          * the tunable/sysctl to alter the default number of threads.
1832          */
1833         retval = taskqueue_start_threads(&be_lun->io_taskqueue,
1834                                          /*num threads*/num_threads,
1835                                          /*priority*/PWAIT,
1836                                          /*thread name*/
1837                                          "%s taskq", be_lun->lunname);
1838
1839         if (retval != 0)
1840                 goto bailout_error;
1841
1842         be_lun->num_threads = num_threads;
1843
1844         mtx_lock(&softc->lock);
1845         softc->num_luns++;
1846         STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links);
1847
1848         mtx_unlock(&softc->lock);
1849
1850         retval = ctl_add_lun(&be_lun->ctl_be_lun);
1851         if (retval != 0) {
1852                 mtx_lock(&softc->lock);
1853                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
1854                               links);
1855                 softc->num_luns--;
1856                 mtx_unlock(&softc->lock);
1857                 snprintf(req->error_str, sizeof(req->error_str),
1858                          "%s: ctl_add_lun() returned error %d, see dmesg for "
1859                         "details", __func__, retval);
1860                 retval = 0;
1861                 goto bailout_error;
1862         }
1863
1864         mtx_lock(&softc->lock);
1865
1866         /*
1867          * Tell the config_status routine that we're waiting so it won't
1868          * clean up the LUN in the event of an error.
1869          */
1870         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
1871
1872         while (be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) {
1873                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
1874                 if (retval == EINTR)
1875                         break;
1876         }
1877         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
1878
1879         if (be_lun->flags & CTL_BE_BLOCK_LUN_CONFIG_ERR) {
1880                 snprintf(req->error_str, sizeof(req->error_str),
1881                          "%s: LUN configuration error, see dmesg for details",
1882                          __func__);
1883                 STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun,
1884                               links);
1885                 softc->num_luns--;
1886                 mtx_unlock(&softc->lock);
1887                 goto bailout_error;
1888         } else {
1889                 params->req_lun_id = be_lun->ctl_be_lun.lun_id;
1890         }
1891
1892         mtx_unlock(&softc->lock);
1893
1894         be_lun->disk_stats = devstat_new_entry("cbb", params->req_lun_id,
1895                                                be_lun->blocksize,
1896                                                DEVSTAT_ALL_SUPPORTED,
1897                                                be_lun->ctl_be_lun.lun_type
1898                                                | DEVSTAT_TYPE_IF_OTHER,
1899                                                DEVSTAT_PRIORITY_OTHER);
1900
1901
1902         req->status = CTL_LUN_OK;
1903
1904         return (retval);
1905
1906 bailout_error:
1907         req->status = CTL_LUN_ERROR;
1908
1909         ctl_be_block_close(be_lun);
1910
1911         free(be_lun->dev_path, M_CTLBLK);
1912         free(be_lun, M_CTLBLK);
1913
1914         return (retval);
1915 }
1916
1917 static int
1918 ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
1919 {
1920         struct ctl_lun_rm_params *params;
1921         struct ctl_be_block_lun *be_lun;
1922         int retval;
1923
1924         params = &req->reqdata.rm;
1925
1926         mtx_lock(&softc->lock);
1927
1928         be_lun = NULL;
1929
1930         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
1931                 if (be_lun->ctl_be_lun.lun_id == params->lun_id)
1932                         break;
1933         }
1934         mtx_unlock(&softc->lock);
1935
1936         if (be_lun == NULL) {
1937                 snprintf(req->error_str, sizeof(req->error_str),
1938                          "%s: LUN %u is not managed by the block backend",
1939                          __func__, params->lun_id);
1940                 goto bailout_error;
1941         }
1942
1943         retval = ctl_disable_lun(&be_lun->ctl_be_lun);
1944
1945         if (retval != 0) {
1946                 snprintf(req->error_str, sizeof(req->error_str),
1947                          "%s: error %d returned from ctl_disable_lun() for "
1948                          "LUN %d", __func__, retval, params->lun_id);
1949                 goto bailout_error;
1950
1951         }
1952
1953         retval = ctl_invalidate_lun(&be_lun->ctl_be_lun);
1954         if (retval != 0) {
1955                 snprintf(req->error_str, sizeof(req->error_str),
1956                          "%s: error %d returned from ctl_invalidate_lun() for "
1957                          "LUN %d", __func__, retval, params->lun_id);
1958                 goto bailout_error;
1959         }
1960
1961         mtx_lock(&softc->lock);
1962
1963         be_lun->flags |= CTL_BE_BLOCK_LUN_WAITING;
1964
1965         while ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
1966                 retval = msleep(be_lun, &softc->lock, PCATCH, "ctlblk", 0);
1967                 if (retval == EINTR)
1968                         break;
1969         }
1970
1971         be_lun->flags &= ~CTL_BE_BLOCK_LUN_WAITING;
1972
1973         if ((be_lun->flags & CTL_BE_BLOCK_LUN_UNCONFIGURED) == 0) {
1974                 snprintf(req->error_str, sizeof(req->error_str),
1975                          "%s: interrupted waiting for LUN to be freed",
1976                          __func__);
1977                 mtx_unlock(&softc->lock);
1978                 goto bailout_error;
1979         }
1980
1981         STAILQ_REMOVE(&softc->lun_list, be_lun, ctl_be_block_lun, links);
1982
1983         softc->num_luns--;
1984         mtx_unlock(&softc->lock);
1985
1986         taskqueue_drain(be_lun->io_taskqueue, &be_lun->io_task);
1987
1988         taskqueue_free(be_lun->io_taskqueue);
1989
1990         ctl_be_block_close(be_lun);
1991
1992         if (be_lun->disk_stats != NULL)
1993                 devstat_remove_entry(be_lun->disk_stats);
1994
1995         uma_zdestroy(be_lun->lun_zone);
1996
1997         free(be_lun->dev_path, M_CTLBLK);
1998
1999         free(be_lun, M_CTLBLK);
2000
2001         req->status = CTL_LUN_OK;
2002
2003         return (0);
2004
2005 bailout_error:
2006
2007         req->status = CTL_LUN_ERROR;
2008
2009         return (0);
2010 }
2011
2012 static int
2013 ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
2014                          struct ctl_lun_req *req)
2015 {
2016         struct vattr vattr;
2017         int error;
2018         struct ctl_lun_modify_params *params;
2019
2020         params = &req->reqdata.modify;
2021
2022         if (params->lun_size_bytes != 0) {
2023                 be_lun->size_bytes = params->lun_size_bytes;
2024         } else  {
2025                 error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
2026                 if (error != 0) {
2027                         snprintf(req->error_str, sizeof(req->error_str),
2028                                  "error calling VOP_GETATTR() for file %s",
2029                                  be_lun->dev_path);
2030                         return (error);
2031                 }
2032
2033                 be_lun->size_bytes = vattr.va_size;
2034         }
2035
2036         return (0);
2037 }
2038
2039 static int
2040 ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
2041                         struct ctl_lun_req *req)
2042 {
2043         struct cdev *dev;
2044         struct cdevsw *devsw;
2045         int error;
2046         struct ctl_lun_modify_params *params;
2047         uint64_t size_bytes;
2048
2049         params = &req->reqdata.modify;
2050
2051         dev = be_lun->vn->v_rdev;
2052         devsw = dev->si_devsw;
2053         if (!devsw->d_ioctl) {
2054                 snprintf(req->error_str, sizeof(req->error_str),
2055                          "%s: no d_ioctl for device %s!", __func__,
2056                          be_lun->dev_path);
2057                 return (ENODEV);
2058         }
2059
2060         error = devsw->d_ioctl(dev, DIOCGMEDIASIZE,
2061                                (caddr_t)&size_bytes, FREAD,
2062                                curthread);
2063         if (error) {
2064                 snprintf(req->error_str, sizeof(req->error_str),
2065                          "%s: error %d returned for DIOCGMEDIASIZE ioctl "
2066                          "on %s!", __func__, error, be_lun->dev_path);
2067                 return (error);
2068         }
2069
2070         if (params->lun_size_bytes != 0) {
2071                 if (params->lun_size_bytes > size_bytes) {
2072                         snprintf(req->error_str, sizeof(req->error_str),
2073                                  "%s: requested LUN size %ju > backing device "
2074                                  "size %ju", __func__,
2075                                  (uintmax_t)params->lun_size_bytes,
2076                                  (uintmax_t)size_bytes);
2077                         return (EINVAL);
2078                 }
2079
2080                 be_lun->size_bytes = params->lun_size_bytes;
2081         } else {
2082                 be_lun->size_bytes = size_bytes;
2083         }
2084
2085         return (0);
2086 }
2087
2088 static int
2089 ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
2090 {
2091         struct ctl_lun_modify_params *params;
2092         struct ctl_be_block_lun *be_lun;
2093         int error;
2094
2095         params = &req->reqdata.modify;
2096
2097         mtx_lock(&softc->lock);
2098
2099         be_lun = NULL;
2100
2101         STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
2102                 if (be_lun->ctl_be_lun.lun_id == params->lun_id)
2103                         break;
2104         }
2105         mtx_unlock(&softc->lock);
2106
2107         if (be_lun == NULL) {
2108                 snprintf(req->error_str, sizeof(req->error_str),
2109                          "%s: LUN %u is not managed by the block backend",
2110                          __func__, params->lun_id);
2111                 goto bailout_error;
2112         }
2113
2114         if (params->lun_size_bytes != 0) {
2115                 if (params->lun_size_bytes < be_lun->blocksize) {
2116                         snprintf(req->error_str, sizeof(req->error_str),
2117                                 "%s: LUN size %ju < blocksize %u", __func__,
2118                                 params->lun_size_bytes, be_lun->blocksize);
2119                         goto bailout_error;
2120                 }
2121         }
2122
2123         vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
2124
2125         if (be_lun->vn->v_type == VREG)
2126                 error = ctl_be_block_modify_file(be_lun, req);
2127         else
2128                 error = ctl_be_block_modify_dev(be_lun, req);
2129
2130         VOP_UNLOCK(be_lun->vn, 0);
2131
2132         if (error != 0)
2133                 goto bailout_error;
2134
2135         be_lun->size_blocks = be_lun->size_bytes >> be_lun->blocksize_shift;
2136
2137         /*
2138          * The maximum LBA is the size - 1.
2139          *
2140          * XXX: Note that this field is being updated without locking,
2141          *      which might cause problems on 32-bit architectures.
2142          */
2143         be_lun->ctl_be_lun.maxlba = be_lun->size_blocks - 1;
2144         ctl_lun_capacity_changed(&be_lun->ctl_be_lun);
2145
2146         /* Tell the user the exact size we ended up using */
2147         params->lun_size_bytes = be_lun->size_bytes;
2148
2149         req->status = CTL_LUN_OK;
2150
2151         return (0);
2152
2153 bailout_error:
2154         req->status = CTL_LUN_ERROR;
2155
2156         return (0);
2157 }
2158
2159 static void
2160 ctl_be_block_lun_shutdown(void *be_lun)
2161 {
2162         struct ctl_be_block_lun *lun;
2163         struct ctl_be_block_softc *softc;
2164
2165         lun = (struct ctl_be_block_lun *)be_lun;
2166
2167         softc = lun->softc;
2168
2169         mtx_lock(&softc->lock);
2170         lun->flags |= CTL_BE_BLOCK_LUN_UNCONFIGURED;
2171         if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2172                 wakeup(lun);
2173         mtx_unlock(&softc->lock);
2174
2175 }
2176
2177 static void
2178 ctl_be_block_lun_config_status(void *be_lun, ctl_lun_config_status status)
2179 {
2180         struct ctl_be_block_lun *lun;
2181         struct ctl_be_block_softc *softc;
2182
2183         lun = (struct ctl_be_block_lun *)be_lun;
2184         softc = lun->softc;
2185
2186         if (status == CTL_LUN_CONFIG_OK) {
2187                 mtx_lock(&softc->lock);
2188                 lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2189                 if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
2190                         wakeup(lun);
2191                 mtx_unlock(&softc->lock);
2192
2193                 /*
2194                  * We successfully added the LUN, attempt to enable it.
2195                  */
2196                 if (ctl_enable_lun(&lun->ctl_be_lun) != 0) {
2197                         printf("%s: ctl_enable_lun() failed!\n", __func__);
2198                         if (ctl_invalidate_lun(&lun->ctl_be_lun) != 0) {
2199                                 printf("%s: ctl_invalidate_lun() failed!\n",
2200                                        __func__);
2201                         }
2202                 }
2203
2204                 return;
2205         }
2206
2207
2208         mtx_lock(&softc->lock);
2209         lun->flags &= ~CTL_BE_BLOCK_LUN_UNCONFIGURED;
2210         lun->flags |= CTL_BE_BLOCK_LUN_CONFIG_ERR;
2211         wakeup(lun);
2212         mtx_unlock(&softc->lock);
2213 }
2214
2215
2216 static int
2217 ctl_be_block_config_write(union ctl_io *io)
2218 {
2219         struct ctl_be_block_lun *be_lun;
2220         struct ctl_be_lun *ctl_be_lun;
2221         int retval;
2222
2223         retval = 0;
2224
2225         DPRINTF("entered\n");
2226
2227         ctl_be_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
2228                 CTL_PRIV_BACKEND_LUN].ptr;
2229         be_lun = (struct ctl_be_block_lun *)ctl_be_lun->be_lun;
2230
2231         switch (io->scsiio.cdb[0]) {
2232         case SYNCHRONIZE_CACHE:
2233         case SYNCHRONIZE_CACHE_16:
2234                 /*
2235                  * The upper level CTL code will filter out any CDBs with
2236                  * the immediate bit set and return the proper error.
2237                  *
2238                  * We don't really need to worry about what LBA range the
2239                  * user asked to be synced out.  When they issue a sync
2240                  * cache command, we'll sync out the whole thing.
2241                  */
2242                 mtx_lock(&be_lun->lock);
2243                 STAILQ_INSERT_TAIL(&be_lun->config_write_queue, &io->io_hdr,
2244                                    links);
2245                 mtx_unlock(&be_lun->lock);
2246                 taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
2247                 break;
2248         case START_STOP_UNIT: {
2249                 struct scsi_start_stop_unit *cdb;
2250
2251                 cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
2252
2253                 if (cdb->how & SSS_START)
2254                         retval = ctl_start_lun(ctl_be_lun);
2255                 else {
2256                         retval = ctl_stop_lun(ctl_be_lun);
2257                         /*
2258                          * XXX KDM Copan-specific offline behavior.
2259                          * Figure out a reasonable way to port this?
2260                          */
2261 #ifdef NEEDTOPORT
2262                         if ((retval == 0)
2263                          && (cdb->byte2 & SSS_ONOFFLINE))
2264                                 retval = ctl_lun_offline(ctl_be_lun);
2265 #endif
2266                 }
2267
2268                 /*
2269                  * In general, the above routines should not fail.  They
2270                  * just set state for the LUN.  So we've got something
2271                  * pretty wrong here if we can't start or stop the LUN.
2272                  */
2273                 if (retval != 0) {
2274                         ctl_set_internal_failure(&io->scsiio,
2275                                                  /*sks_valid*/ 1,
2276                                                  /*retry_count*/ 0xf051);
2277                         retval = CTL_RETVAL_COMPLETE;
2278                 } else {
2279                         ctl_set_success(&io->scsiio);
2280                 }
2281                 ctl_config_write_done(io);
2282                 break;
2283         }
2284         default:
2285                 ctl_set_invalid_opcode(&io->scsiio);
2286                 ctl_config_write_done(io);
2287                 retval = CTL_RETVAL_COMPLETE;
2288                 break;
2289         }
2290
2291         return (retval);
2292
2293 }
2294
2295 static int
2296 ctl_be_block_config_read(union ctl_io *io)
2297 {
2298         return (0);
2299 }
2300
2301 static int
2302 ctl_be_block_lun_info(void *be_lun, struct sbuf *sb)
2303 {
2304         struct ctl_be_block_lun *lun;
2305         int retval;
2306
2307         lun = (struct ctl_be_block_lun *)be_lun;
2308         retval = 0;
2309
2310         retval = sbuf_printf(sb, "<num_threads>");
2311
2312         if (retval != 0)
2313                 goto bailout;
2314
2315         retval = sbuf_printf(sb, "%d", lun->num_threads);
2316
2317         if (retval != 0)
2318                 goto bailout;
2319
2320         retval = sbuf_printf(sb, "</num_threads>");
2321
2322         /*
2323          * For processor devices, we don't have a path variable.
2324          */
2325         if ((retval != 0)
2326          || (lun->dev_path == NULL))
2327                 goto bailout;
2328
2329         retval = sbuf_printf(sb, "<file>");
2330
2331         if (retval != 0)
2332                 goto bailout;
2333
2334         retval = ctl_sbuf_printf_esc(sb, lun->dev_path);
2335
2336         if (retval != 0)
2337                 goto bailout;
2338
2339         retval = sbuf_printf(sb, "</file>\n");
2340
2341 bailout:
2342
2343         return (retval);
2344 }
2345
2346 int
2347 ctl_be_block_init(void)
2348 {
2349         struct ctl_be_block_softc *softc;
2350         int retval;
2351
2352         softc = &backend_block_softc;
2353         retval = 0;
2354
2355         mtx_init(&softc->lock, "ctlblk", NULL, MTX_DEF);
2356         STAILQ_INIT(&softc->beio_free_queue);
2357         STAILQ_INIT(&softc->disk_list);
2358         STAILQ_INIT(&softc->lun_list);
2359         ctl_grow_beio(softc, 200);
2360
2361         return (retval);
2362 }