usr.sbin/camdd/camdd.c

   1 /*-
   2  * Copyright (c) 1997-2007 Kenneth D. Merry
   3  * Copyright (c) 2013, 2014, 2015 Spectra Logic Corporation
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions, and the following disclaimer,
  11  *    without modification.
  12  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  13  *    substantially similar to the "NO WARRANTY" disclaimer below
  14  *    ("Disclaimer") and any redistribution must be conditioned upon
  15  *    including a substantially similar Disclaimer requirement for further
  16  *    binary redistribution.
  17  *
  18  * NO WARRANTY
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  22  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  27  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  28  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29  * POSSIBILITY OF SUCH DAMAGES.
  30  *
  31  * Authors: Ken Merry           (Spectra Logic Corporation)
  32  */
  33
  34 /*
  35  * This is eventually intended to be:
  36  * - A basic data transfer/copy utility
  37  * - A simple benchmark utility
  38  * - An example of how to use the asynchronous pass(4) driver interface.
  39  */
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include <sys/ioctl.h>
  44 #include <sys/stdint.h>
  45 #include <sys/types.h>
  46 #include <sys/endian.h>
  47 #include <sys/param.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/stat.h>
  50 #include <sys/event.h>
  51 #include <sys/time.h>
  52 #include <sys/uio.h>
  53 #include <vm/vm.h>
  54 #include <sys/bus.h>
  55 #include <sys/bus_dma.h>
  56 #include <sys/mtio.h>
  57 #include <sys/conf.h>
  58 #include <sys/disk.h>
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <semaphore.h>
  63 #include <string.h>
  64 #include <unistd.h>
  65 #include <inttypes.h>
  66 #include <limits.h>
  67 #include <fcntl.h>
  68 #include <ctype.h>
  69 #include <err.h>
  70 #include <libutil.h>
  71 #include <pthread.h>
  72 #include <assert.h>
  73 #include <bsdxml.h>
  74
  75 #include <cam/cam.h>
  76 #include <cam/cam_debug.h>
  77 #include <cam/cam_ccb.h>
  78 #include <cam/scsi/scsi_all.h>
  79 #include <cam/scsi/scsi_da.h>
  80 #include <cam/scsi/scsi_pass.h>
  81 #include <cam/scsi/scsi_message.h>
  82 #include <cam/scsi/smp_all.h>
  83 #include <cam/nvme/nvme_all.h>
  84 #include <camlib.h>
  85 #include <mtlib.h>
  86 #include <zlib.h>
  87
  88 typedef enum {
  89         CAMDD_CMD_NONE          = 0x00000000,
  90         CAMDD_CMD_HELP          = 0x00000001,
  91         CAMDD_CMD_WRITE         = 0x00000002,
  92         CAMDD_CMD_READ          = 0x00000003
  93 } camdd_cmdmask;
  94
  95 typedef enum {
  96         CAMDD_ARG_NONE          = 0x00000000,
  97         CAMDD_ARG_VERBOSE       = 0x00000001,
  98         CAMDD_ARG_DEVICE        = 0x00000002,
  99         CAMDD_ARG_BUS           = 0x00000004,
 100         CAMDD_ARG_TARGET        = 0x00000008,
 101         CAMDD_ARG_LUN           = 0x00000010,
 102         CAMDD_ARG_UNIT          = 0x00000020,
 103         CAMDD_ARG_TIMEOUT       = 0x00000040,
 104         CAMDD_ARG_ERR_RECOVER   = 0x00000080,
 105         CAMDD_ARG_RETRIES       = 0x00000100
 106 } camdd_argmask;
 107
 108 typedef enum {
 109         CAMDD_DEV_NONE          = 0x00,
 110         CAMDD_DEV_PASS          = 0x01,
 111         CAMDD_DEV_FILE          = 0x02
 112 } camdd_dev_type;
 113
 114 struct camdd_io_opts {
 115         camdd_dev_type  dev_type;
 116         char            *dev_name;
 117         uint64_t        blocksize;
 118         uint64_t        queue_depth;
 119         uint64_t        offset;
 120         int             min_cmd_size;
 121         int             write_dev;
 122         uint64_t        debug;
 123 };
 124
 125 typedef enum {
 126         CAMDD_BUF_NONE,
 127         CAMDD_BUF_DATA,
 128         CAMDD_BUF_INDIRECT
 129 } camdd_buf_type;
 130
 131 struct camdd_buf_indirect {
 132         /*
 133          * Pointer to the source buffer.
 134          */
 135         struct camdd_buf *src_buf;
 136
 137         /*
 138          * Offset into the source buffer, in bytes.
 139          */
 140         uint64_t          offset;
 141         /*
 142          * Pointer to the starting point in the source buffer.
 143          */
 144         uint8_t          *start_ptr;
 145
 146         /*
 147          * Length of this chunk in bytes.
 148          */
 149         size_t            len;
 150 };
 151
 152 struct camdd_buf_data {
 153         /*
 154          * Buffer allocated when we allocate this camdd_buf.  This should
 155          * be the size of the blocksize for this device.
 156          */
 157         uint8_t                 *buf;
 158
 159         /*
 160          * The amount of backing store allocated in buf.  Generally this
 161          * will be the blocksize of the device.
 162          */
 163         uint32_t                 alloc_len;
 164
 165         /*
 166          * The amount of data that was put into the buffer (on reads) or
 167          * the amount of data we have put onto the src_list so far (on
 168          * writes).
 169          */
 170         uint32_t                 fill_len;
 171
 172         /*
 173          * The amount of data that was not transferred.
 174          */
 175         uint32_t                 resid;
 176
 177         /*
 178          * Starting byte offset on the reader.
 179          */
 180         uint64_t                 src_start_offset;
 181
 182         /*
 183          * CCB used for pass(4) device targets.
 184          */
 185         union ccb                ccb;
 186
 187         /*
 188          * Number of scatter/gather segments.
 189          */
 190         int                      sg_count;
 191
 192         /*
 193          * Set if we had to tack on an extra buffer to round the transfer
 194          * up to a sector size.
 195          */
 196         int                      extra_buf;
 197
 198         /*
 199          * Scatter/gather list used generally when we're the writer for a
 200          * pass(4) device.
 201          */
 202         bus_dma_segment_t       *segs;
 203
 204         /*
 205          * Scatter/gather list used generally when we're the writer for a
 206          * file or block device;
 207          */
 208         struct iovec            *iovec;
 209 };
 210
 211 union camdd_buf_types {
 212         struct camdd_buf_indirect       indirect;
 213         struct camdd_buf_data           data;
 214 };
 215
 216 typedef enum {
 217         CAMDD_STATUS_NONE,
 218         CAMDD_STATUS_OK,
 219         CAMDD_STATUS_SHORT_IO,
 220         CAMDD_STATUS_EOF,
 221         CAMDD_STATUS_ERROR
 222 } camdd_buf_status;
 223
 224 struct camdd_buf {
 225         camdd_buf_type           buf_type;
 226         union camdd_buf_types    buf_type_spec;
 227
 228         camdd_buf_status         status;
 229
 230         uint64_t                 lba;
 231         size_t                   len;
 232
 233         /*
 234          * A reference count of how many indirect buffers point to this
 235          * buffer.
 236          */
 237         int                      refcount;
 238
 239         /*
 240          * A link back to our parent device.
 241          */
 242         struct camdd_dev        *dev;
 243         STAILQ_ENTRY(camdd_buf)  links;
 244         STAILQ_ENTRY(camdd_buf)  work_links;
 245
 246         /*
 247          * A count of the buffers on the src_list.
 248          */
 249         int                      src_count;
 250
 251         /*
 252          * List of buffers from our partner thread that are the components
 253          * of this buffer for the I/O.  Uses src_links.
 254          */
 255         STAILQ_HEAD(,camdd_buf)  src_list;
 256         STAILQ_ENTRY(camdd_buf)  src_links;
 257 };
 258
 259 #define NUM_DEV_TYPES   2
 260
 261 struct camdd_dev_pass {
 262         int                      scsi_dev_type;
 263         int                      protocol;
 264         struct cam_device       *dev;
 265         uint64_t                 max_sector;
 266         uint32_t                 block_len;
 267         uint32_t                 cpi_maxio;
 268 };
 269
 270 typedef enum {
 271         CAMDD_FILE_NONE,
 272         CAMDD_FILE_REG,
 273         CAMDD_FILE_STD,
 274         CAMDD_FILE_PIPE,
 275         CAMDD_FILE_DISK,
 276         CAMDD_FILE_TAPE,
 277         CAMDD_FILE_TTY,
 278         CAMDD_FILE_MEM
 279 } camdd_file_type;
 280
 281 typedef enum {
 282         CAMDD_FF_NONE           = 0x00,
 283         CAMDD_FF_CAN_SEEK       = 0x01
 284 } camdd_file_flags;
 285
 286 struct camdd_dev_file {
 287         int                      fd;
 288         struct stat              sb;
 289         char                     filename[MAXPATHLEN + 1];
 290         camdd_file_type          file_type;
 291         camdd_file_flags         file_flags;
 292         uint8_t                 *tmp_buf;
 293 };
 294
 295 struct camdd_dev_block {
 296         int                      fd;
 297         uint64_t                 size_bytes;
 298         uint32_t                 block_len;
 299 };
 300
 301 union camdd_dev_spec {
 302         struct camdd_dev_pass   pass;
 303         struct camdd_dev_file   file;
 304         struct camdd_dev_block  block;
 305 };
 306
 307 typedef enum {
 308         CAMDD_DEV_FLAG_NONE             = 0x00,
 309         CAMDD_DEV_FLAG_EOF              = 0x01,
 310         CAMDD_DEV_FLAG_PEER_EOF         = 0x02,
 311         CAMDD_DEV_FLAG_ACTIVE           = 0x04,
 312         CAMDD_DEV_FLAG_EOF_SENT         = 0x08,
 313         CAMDD_DEV_FLAG_EOF_QUEUED       = 0x10
 314 } camdd_dev_flags;
 315
 316 struct camdd_dev {
 317         camdd_dev_type           dev_type;
 318         union camdd_dev_spec     dev_spec;
 319         camdd_dev_flags          flags;
 320         char                     device_name[MAXPATHLEN+1];
 321         uint32_t                 blocksize;
 322         uint32_t                 sector_size;
 323         uint64_t                 max_sector;
 324         uint64_t                 sector_io_limit;
 325         int                      min_cmd_size;
 326         int                      write_dev;
 327         int                      retry_count;
 328         int                      io_timeout;
 329         int                      debug;
 330         uint64_t                 start_offset_bytes;
 331         uint64_t                 next_io_pos_bytes;
 332         uint64_t                 next_peer_pos_bytes;
 333         uint64_t                 next_completion_pos_bytes;
 334         uint64_t                 peer_bytes_queued;
 335         uint64_t                 bytes_transferred;
 336         uint32_t                 target_queue_depth;
 337         uint32_t                 cur_active_io;
 338         uint8_t                 *extra_buf;
 339         uint32_t                 extra_buf_len;
 340         struct camdd_dev        *peer_dev;
 341         pthread_mutex_t          mutex;
 342         pthread_cond_t           cond;
 343         int                      kq;
 344
 345         int                      (*run)(struct camdd_dev *dev);
 346         int                      (*fetch)(struct camdd_dev *dev);
 347
 348         /*
 349          * Buffers that are available for I/O.  Uses links.
 350          */
 351         STAILQ_HEAD(,camdd_buf)  free_queue;
 352
 353         /*
 354          * Free indirect buffers.  These are used for breaking a large
 355          * buffer into multiple pieces.
 356          */
 357         STAILQ_HEAD(,camdd_buf)  free_indirect_queue;
 358
 359         /*
 360          * Buffers that have been queued to the kernel.  Uses links.
 361          */
 362         STAILQ_HEAD(,camdd_buf)  active_queue;
 363
 364         /*
 365          * Will generally contain one of our buffers that is waiting for enough
 366          * I/O from our partner thread to be able to execute.  This will
 367          * generally happen when our per-I/O-size is larger than the
 368          * partner thread's per-I/O-size.  Uses links.
 369          */
 370         STAILQ_HEAD(,camdd_buf)  pending_queue;
 371
 372         /*
 373          * Number of buffers on the pending queue
 374          */
 375         int                      num_pending_queue;
 376
 377         /*
 378          * Buffers that are filled and ready to execute.  This is used when
 379          * our partner (reader) thread sends us blocks that are larger than
 380          * our blocksize, and so we have to split them into multiple pieces.
 381          */
 382         STAILQ_HEAD(,camdd_buf)  run_queue;
 383
 384         /*
 385          * Number of buffers on the run queue.
 386          */
 387         int                      num_run_queue;
 388
 389         STAILQ_HEAD(,camdd_buf)  reorder_queue;
 390
 391         int                      num_reorder_queue;
 392
 393         /*
 394          * Buffers that have been queued to us by our partner thread
 395          * (generally the reader thread) to be written out.  Uses
 396          * work_links.
 397          */
 398         STAILQ_HEAD(,camdd_buf)  work_queue;
 399
 400         /*
 401          * Buffers that have been completed by our partner thread.  Uses
 402          * work_links.
 403          */
 404         STAILQ_HEAD(,camdd_buf)  peer_done_queue;
 405
 406         /*
 407          * Number of buffers on the peer done queue.
 408          */
 409         uint32_t                 num_peer_done_queue;
 410
 411         /*
 412          * A list of buffers that we have queued to our peer thread.  Uses
 413          * links.
 414          */
 415         STAILQ_HEAD(,camdd_buf)  peer_work_queue;
 416
 417         /*
 418          * Number of buffers on the peer work queue.
 419          */
 420         uint32_t                 num_peer_work_queue;
 421 };
 422
 423 static sem_t camdd_sem;
 424 static sig_atomic_t need_exit = 0;
 425 static sig_atomic_t error_exit = 0;
 426 static sig_atomic_t need_status = 0;
 427
 428 #ifndef min
 429 #define min(a, b) (a < b) ? a : b
 430 #endif
 431
 432
 433 /* Generically useful offsets into the peripheral private area */
 434 #define ppriv_ptr0 periph_priv.entries[0].ptr
 435 #define ppriv_ptr1 periph_priv.entries[1].ptr
 436 #define ppriv_field0 periph_priv.entries[0].field
 437 #define ppriv_field1 periph_priv.entries[1].field
 438
 439 #define ccb_buf ppriv_ptr0
 440
 441 #define CAMDD_FILE_DEFAULT_BLOCK        524288
 442 #define CAMDD_FILE_DEFAULT_DEPTH        1
 443 #define CAMDD_PASS_MAX_BLOCK            1048576
 444 #define CAMDD_PASS_DEFAULT_DEPTH        6
 445 #define CAMDD_PASS_RW_TIMEOUT           60 * 1000
 446
 447 static int parse_btl(char *tstr, int *bus, int *target, int *lun,
 448                      camdd_argmask *arglst);
 449 void camdd_free_dev(struct camdd_dev *dev);
 450 struct camdd_dev *camdd_alloc_dev(camdd_dev_type dev_type,
 451                                   struct kevent *new_ke, int num_ke,
 452                                   int retry_count, int timeout);
 453 static struct camdd_buf *camdd_alloc_buf(struct camdd_dev *dev,
 454                                          camdd_buf_type buf_type);
 455 void camdd_release_buf(struct camdd_buf *buf);
 456 struct camdd_buf *camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type);
 457 int camdd_buf_sg_create(struct camdd_buf *buf, int iovec,
 458                         uint32_t sector_size, uint32_t *num_sectors_used,
 459                         int *double_buf_needed);
 460 uint32_t camdd_buf_get_len(struct camdd_buf *buf);
 461 void camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf);
 462 int camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize,
 463                      uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran);
 464 int camdd_probe_pass_scsi(struct cam_device *cam_dev, union ccb *ccb,
 465          camdd_argmask arglist, int probe_retry_count,
 466          int probe_timeout, uint64_t *maxsector, uint32_t *block_len);
 467 int camdd_probe_pass_nvme(struct cam_device *cam_dev, union ccb *ccb,
 468          camdd_argmask arglist, int probe_retry_count,
 469          int probe_timeout, uint64_t *maxsector, uint32_t *block_len);
 470 struct camdd_dev *camdd_probe_file(int fd, struct camdd_io_opts *io_opts,
 471                                    int retry_count, int timeout);
 472 struct camdd_dev *camdd_probe_pass(struct cam_device *cam_dev,
 473                                    struct camdd_io_opts *io_opts,
 474                                    camdd_argmask arglist, int probe_retry_count,
 475                                    int probe_timeout, int io_retry_count,
 476                                    int io_timeout);
 477 void nvme_read_write(struct ccb_nvmeio *nvmeio, uint32_t retries,
 478                 void (*cbfcnp)(struct cam_periph *, union ccb *),
 479                 uint32_t nsid, int readop, uint64_t lba,
 480                 uint32_t block_count, uint8_t *data_ptr, uint32_t dxfer_len,
 481                 uint32_t timeout);
 482 void *camdd_file_worker(void *arg);
 483 camdd_buf_status camdd_ccb_status(union ccb *ccb, int protocol);
 484 int camdd_get_cgd(struct cam_device *device, struct ccb_getdev *cgd);
 485 int camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf);
 486 int camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf);
 487 void camdd_peer_done(struct camdd_buf *buf);
 488 void camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf,
 489                         int *error_count);
 490 int camdd_pass_fetch(struct camdd_dev *dev);
 491 int camdd_file_run(struct camdd_dev *dev);
 492 int camdd_pass_run(struct camdd_dev *dev);
 493 int camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len);
 494 int camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf);
 495 void camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth,
 496                      uint32_t *peer_depth, uint32_t *our_bytes,
 497                      uint32_t *peer_bytes);
 498 void *camdd_worker(void *arg);
 499 void camdd_sig_handler(int sig);
 500 void camdd_print_status(struct camdd_dev *camdd_dev,
 501                         struct camdd_dev *other_dev,
 502                         struct timespec *start_time);
 503 int camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts,
 504              uint64_t max_io, int retry_count, int timeout);
 505 int camdd_parse_io_opts(char *args, int is_write,
 506                         struct camdd_io_opts *io_opts);
 507 void usage(void);
 508
 509 /*
 510  * Parse out a bus, or a bus, target and lun in the following
 511  * format:
 512  * bus
 513  * bus:target
 514  * bus:target:lun
 515  *
 516  * Returns the number of parsed components, or 0.
 517  */
 518 static int
 519 parse_btl(char *tstr, int *bus, int *target, int *lun, camdd_argmask *arglst)
 520 {
 521         char *tmpstr;
 522         int convs = 0;
 523
 524         while (isspace(*tstr) && (*tstr != '\0'))
 525                 tstr++;
 526
 527         tmpstr = (char *)strtok(tstr, ":");
 528         if ((tmpstr != NULL) && (*tmpstr != '\0')) {
 529                 *bus = strtol(tmpstr, NULL, 0);
 530                 *arglst |= CAMDD_ARG_BUS;
 531                 convs++;
 532                 tmpstr = (char *)strtok(NULL, ":");
 533                 if ((tmpstr != NULL) && (*tmpstr != '\0')) {
 534                         *target = strtol(tmpstr, NULL, 0);
 535                         *arglst |= CAMDD_ARG_TARGET;
 536                         convs++;
 537                         tmpstr = (char *)strtok(NULL, ":");
 538                         if ((tmpstr != NULL) && (*tmpstr != '\0')) {
 539                                 *lun = strtol(tmpstr, NULL, 0);
 540                                 *arglst |= CAMDD_ARG_LUN;
 541                                 convs++;
 542                         }
 543                 }
 544         }
 545
 546         return convs;
 547 }
 548
 549 /*
 550  * XXX KDM clean up and free all of the buffers on the queue!
 551  */
 552 void
 553 camdd_free_dev(struct camdd_dev *dev)
 554 {
 555         if (dev == NULL)
 556                 return;
 557
 558         switch (dev->dev_type) {
 559         case CAMDD_DEV_FILE: {
 560                 struct camdd_dev_file *file_dev = &dev->dev_spec.file;
 561
 562                 if (file_dev->fd != -1)
 563                         close(file_dev->fd);
 564                 free(file_dev->tmp_buf);
 565                 break;
 566         }
 567         case CAMDD_DEV_PASS: {
 568                 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass;
 569
 570                 if (pass_dev->dev != NULL)
 571                         cam_close_device(pass_dev->dev);
 572                 break;
 573         }
 574         default:
 575                 break;
 576         }
 577
 578         free(dev);
 579 }
 580
 581 struct camdd_dev *
 582 camdd_alloc_dev(camdd_dev_type dev_type, struct kevent *new_ke, int num_ke,
 583                 int retry_count, int timeout)
 584 {
 585         struct camdd_dev *dev = NULL;
 586         struct kevent *ke;
 587         size_t ke_size;
 588         int retval = 0;
 589
 590         dev = calloc(1, sizeof(*dev));
 591         if (dev == NULL) {
 592                 warn("%s: unable to malloc %zu bytes", __func__, sizeof(*dev));
 593                 goto bailout;
 594         }
 595
 596         dev->dev_type = dev_type;
 597         dev->io_timeout = timeout;
 598         dev->retry_count = retry_count;
 599         STAILQ_INIT(&dev->free_queue);
 600         STAILQ_INIT(&dev->free_indirect_queue);
 601         STAILQ_INIT(&dev->active_queue);
 602         STAILQ_INIT(&dev->pending_queue);
 603         STAILQ_INIT(&dev->run_queue);
 604         STAILQ_INIT(&dev->reorder_queue);
 605         STAILQ_INIT(&dev->work_queue);
 606         STAILQ_INIT(&dev->peer_done_queue);
 607         STAILQ_INIT(&dev->peer_work_queue);
 608         retval = pthread_mutex_init(&dev->mutex, NULL);
 609         if (retval != 0) {
 610                 warnc(retval, "%s: failed to initialize mutex", __func__);
 611                 goto bailout;
 612         }
 613
 614         retval = pthread_cond_init(&dev->cond, NULL);
 615         if (retval != 0) {
 616                 warnc(retval, "%s: failed to initialize condition variable",
 617                       __func__);
 618                 goto bailout;
 619         }
 620
 621         dev->kq = kqueue();
 622         if (dev->kq == -1) {
 623                 warn("%s: Unable to create kqueue", __func__);
 624                 goto bailout;
 625         }
 626
 627         ke_size = sizeof(struct kevent) * (num_ke + 4);
 628         ke = calloc(1, ke_size);
 629         if (ke == NULL) {
 630                 warn("%s: unable to malloc %zu bytes", __func__, ke_size);
 631                 goto bailout;
 632         }
 633         if (num_ke > 0)
 634                 bcopy(new_ke, ke, num_ke * sizeof(struct kevent));
 635
 636         EV_SET(&ke[num_ke++], (uintptr_t)&dev->work_queue, EVFILT_USER,
 637                EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0);
 638         EV_SET(&ke[num_ke++], (uintptr_t)&dev->peer_done_queue, EVFILT_USER,
 639                EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0);
 640         EV_SET(&ke[num_ke++], SIGINFO, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0);
 641         EV_SET(&ke[num_ke++], SIGINT, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0);
 642
 643         retval = kevent(dev->kq, ke, num_ke, NULL, 0, NULL);
 644         if (retval == -1) {
 645                 warn("%s: Unable to register kevents", __func__);
 646                 goto bailout;
 647         }
 648
 649
 650         return (dev);
 651
 652 bailout:
 653         free(dev);
 654
 655         return (NULL);
 656 }
 657
 658 static struct camdd_buf *
 659 camdd_alloc_buf(struct camdd_dev *dev, camdd_buf_type buf_type)
 660 {
 661         struct camdd_buf *buf = NULL;
 662         uint8_t *data_ptr = NULL;
 663
 664         /*
 665          * We only need to allocate data space for data buffers.
 666          */
 667         switch (buf_type) {
 668         case CAMDD_BUF_DATA:
 669                 data_ptr = malloc(dev->blocksize);
 670                 if (data_ptr == NULL) {
 671                         warn("unable to allocate %u bytes", dev->blocksize);
 672                         goto bailout_error;
 673                 }
 674                 break;
 675         default:
 676                 break;
 677         }
 678
 679         buf = calloc(1, sizeof(*buf));
 680         if (buf == NULL) {
 681                 warn("unable to allocate %zu bytes", sizeof(*buf));
 682                 goto bailout_error;
 683         }
 684
 685         buf->buf_type = buf_type;
 686         buf->dev = dev;
 687         switch (buf_type) {
 688         case CAMDD_BUF_DATA: {
 689                 struct camdd_buf_data *data;
 690
 691                 data = &buf->buf_type_spec.data;
 692
 693                 data->alloc_len = dev->blocksize;
 694                 data->buf = data_ptr;
 695                 break;
 696         }
 697         case CAMDD_BUF_INDIRECT:
 698                 break;
 699         default:
 700                 break;
 701         }
 702         STAILQ_INIT(&buf->src_list);
 703
 704         return (buf);
 705
 706 bailout_error:
 707         free(data_ptr);
 708
 709         return (NULL);
 710 }
 711
 712 void
 713 camdd_release_buf(struct camdd_buf *buf)
 714 {
 715         struct camdd_dev *dev;
 716
 717         dev = buf->dev;
 718
 719         switch (buf->buf_type) {
 720         case CAMDD_BUF_DATA: {
 721                 struct camdd_buf_data *data;
 722
 723                 data = &buf->buf_type_spec.data;
 724
 725                 if (data->segs != NULL) {
 726                         if (data->extra_buf != 0) {
 727                                 void *extra_buf;
 728
 729                                 extra_buf = (void *)
 730                                     data->segs[data->sg_count - 1].ds_addr;
 731                                 free(extra_buf);
 732                                 data->extra_buf = 0;
 733                         }
 734                         free(data->segs);
 735                         data->segs = NULL;
 736                         data->sg_count = 0;
 737                 } else if (data->iovec != NULL) {
 738                         if (data->extra_buf != 0) {
 739                                 free(data->iovec[data->sg_count - 1].iov_base);
 740                                 data->extra_buf = 0;
 741                         }
 742                         free(data->iovec);
 743                         data->iovec = NULL;
 744                         data->sg_count = 0;
 745                 }
 746                 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
 747                 break;
 748         }
 749         case CAMDD_BUF_INDIRECT:
 750                 STAILQ_INSERT_TAIL(&dev->free_indirect_queue, buf, links);
 751                 break;
 752         default:
 753                 err(1, "%s: Invalid buffer type %d for released buffer",
 754                     __func__, buf->buf_type);
 755                 break;
 756         }
 757 }
 758
 759 struct camdd_buf *
 760 camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type)
 761 {
 762         struct camdd_buf *buf = NULL;
 763
 764         switch (buf_type) {
 765         case CAMDD_BUF_DATA:
 766                 buf = STAILQ_FIRST(&dev->free_queue);
 767                 if (buf != NULL) {
 768                         struct camdd_buf_data *data;
 769                         uint8_t *data_ptr;
 770                         uint32_t alloc_len;
 771
 772                         STAILQ_REMOVE_HEAD(&dev->free_queue, links);
 773                         data = &buf->buf_type_spec.data;
 774                         data_ptr = data->buf;
 775                         alloc_len = data->alloc_len;
 776                         bzero(buf, sizeof(*buf));
 777                         data->buf = data_ptr;
 778                         data->alloc_len = alloc_len;
 779                 }
 780                 break;
 781         case CAMDD_BUF_INDIRECT:
 782                 buf = STAILQ_FIRST(&dev->free_indirect_queue);
 783                 if (buf != NULL) {
 784                         STAILQ_REMOVE_HEAD(&dev->free_indirect_queue, links);
 785
 786                         bzero(buf, sizeof(*buf));
 787                 }
 788                 break;
 789         default:
 790                 warnx("Unknown buffer type %d requested", buf_type);
 791                 break;
 792         }
 793
 794
 795         if (buf == NULL)
 796                 return (camdd_alloc_buf(dev, buf_type));
 797         else {
 798                 STAILQ_INIT(&buf->src_list);
 799                 buf->dev = dev;
 800                 buf->buf_type = buf_type;
 801
 802                 return (buf);
 803         }
 804 }
 805
 806 int
 807 camdd_buf_sg_create(struct camdd_buf *buf, int iovec, uint32_t sector_size,
 808                     uint32_t *num_sectors_used, int *double_buf_needed)
 809 {
 810         struct camdd_buf *tmp_buf;
 811         struct camdd_buf_data *data;
 812         uint8_t *extra_buf = NULL;
 813         size_t extra_buf_len = 0;
 814         int extra_buf_attached = 0;
 815         int i, retval = 0;
 816
 817         data = &buf->buf_type_spec.data;
 818
 819         data->sg_count = buf->src_count;
 820         /*
 821          * Compose a scatter/gather list from all of the buffers in the list.
 822          * If the length of the buffer isn't a multiple of the sector size,
 823          * we'll have to add an extra buffer.  This should only happen
 824          * at the end of a transfer.
 825          */
 826         if ((data->fill_len % sector_size) != 0) {
 827                 extra_buf_len = sector_size - (data->fill_len % sector_size);
 828                 extra_buf = calloc(extra_buf_len, 1);
 829                 if (extra_buf == NULL) {
 830                         warn("%s: unable to allocate %zu bytes for extra "
 831                             "buffer space", __func__, extra_buf_len);
 832                         retval = 1;
 833                         goto bailout;
 834                 }
 835                 data->extra_buf = 1;
 836                 data->sg_count++;
 837         }
 838         if (iovec == 0) {
 839                 data->segs = calloc(data->sg_count, sizeof(bus_dma_segment_t));
 840                 if (data->segs == NULL) {
 841                         warn("%s: unable to allocate %zu bytes for S/G list",
 842                             __func__, sizeof(bus_dma_segment_t) *
 843                             data->sg_count);
 844                         retval = 1;
 845                         goto bailout;
 846                 }
 847
 848         } else {
 849                 data->iovec = calloc(data->sg_count, sizeof(struct iovec));
 850                 if (data->iovec == NULL) {
 851                         warn("%s: unable to allocate %zu bytes for S/G list",
 852                             __func__, sizeof(struct iovec) * data->sg_count);
 853                         retval = 1;
 854                         goto bailout;
 855                 }
 856         }
 857
 858         for (i = 0, tmp_buf = STAILQ_FIRST(&buf->src_list);
 859              i < buf->src_count && tmp_buf != NULL; i++,
 860              tmp_buf = STAILQ_NEXT(tmp_buf, src_links)) {
 861
 862                 if (tmp_buf->buf_type == CAMDD_BUF_DATA) {
 863                         struct camdd_buf_data *tmp_data;
 864
 865                         tmp_data = &tmp_buf->buf_type_spec.data;
 866                         if (iovec == 0) {
 867                                 data->segs[i].ds_addr =
 868                                     (bus_addr_t) tmp_data->buf;
 869                                 data->segs[i].ds_len = tmp_data->fill_len -
 870                                     tmp_data->resid;
 871                         } else {
 872                                 data->iovec[i].iov_base = tmp_data->buf;
 873                                 data->iovec[i].iov_len = tmp_data->fill_len -
 874                                     tmp_data->resid;
 875                         }
 876                         if (((tmp_data->fill_len - tmp_data->resid) %
 877                              sector_size) != 0)
 878                                 *double_buf_needed = 1;
 879                 } else {
 880                         struct camdd_buf_indirect *tmp_ind;
 881
 882                         tmp_ind = &tmp_buf->buf_type_spec.indirect;
 883                         if (iovec == 0) {
 884                                 data->segs[i].ds_addr =
 885                                     (bus_addr_t)tmp_ind->start_ptr;
 886                                 data->segs[i].ds_len = tmp_ind->len;
 887                         } else {
 888                                 data->iovec[i].iov_base = tmp_ind->start_ptr;
 889                                 data->iovec[i].iov_len = tmp_ind->len;
 890                         }
 891                         if ((tmp_ind->len % sector_size) != 0)
 892                                 *double_buf_needed = 1;
 893                 }
 894         }
 895
 896         if (extra_buf != NULL) {
 897                 if (iovec == 0) {
 898                         data->segs[i].ds_addr = (bus_addr_t)extra_buf;
 899                         data->segs[i].ds_len = extra_buf_len;
 900                 } else {
 901                         data->iovec[i].iov_base = extra_buf;
 902                         data->iovec[i].iov_len = extra_buf_len;
 903                 }
 904                 extra_buf_attached = 1;
 905                 i++;
 906         }
 907         if ((tmp_buf != NULL) || (i != data->sg_count)) {
 908                 warnx("buffer source count does not match "
 909                       "number of buffers in list!");
 910                 retval = 1;
 911                 goto bailout;
 912         }
 913
 914 bailout:
 915         if (retval == 0) {
 916                 *num_sectors_used = (data->fill_len + extra_buf_len) /
 917                     sector_size;
 918         } else if (extra_buf_attached == 0) {
 919                 /*
 920                  * If extra_buf isn't attached yet, we need to free it
 921                  * to avoid leaking.
 922                  */
 923                 free(extra_buf);
 924                 data->extra_buf = 0;
 925                 data->sg_count--;
 926         }
 927         return (retval);
 928 }
 929
 930 uint32_t
 931 camdd_buf_get_len(struct camdd_buf *buf)
 932 {
 933         uint32_t len = 0;
 934
 935         if (buf->buf_type != CAMDD_BUF_DATA) {
 936                 struct camdd_buf_indirect *indirect;
 937
 938                 indirect = &buf->buf_type_spec.indirect;
 939                 len = indirect->len;
 940         } else {
 941                 struct camdd_buf_data *data;
 942
 943                 data = &buf->buf_type_spec.data;
 944                 len = data->fill_len;
 945         }
 946
 947         return (len);
 948 }
 949
 950 void
 951 camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf)
 952 {
 953         struct camdd_buf_data *data;
 954
 955         assert(buf->buf_type == CAMDD_BUF_DATA);
 956
 957         data = &buf->buf_type_spec.data;
 958
 959         STAILQ_INSERT_TAIL(&buf->src_list, child_buf, src_links);
 960         buf->src_count++;
 961
 962         data->fill_len += camdd_buf_get_len(child_buf);
 963 }
 964
 965 typedef enum {
 966         CAMDD_TS_MAX_BLK,
 967         CAMDD_TS_MIN_BLK,
 968         CAMDD_TS_BLK_GRAN,
 969         CAMDD_TS_EFF_IOSIZE
 970 } camdd_status_item_index;
 971
 972 static struct camdd_status_items {
 973         const char *name;
 974         struct mt_status_entry *entry;
 975 } req_status_items[] = {
 976         { "max_blk", NULL },
 977         { "min_blk", NULL },
 978         { "blk_gran", NULL },
 979         { "max_effective_iosize", NULL }
 980 };
 981
 982 int
 983 camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize,
 984                  uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran)
 985 {
 986         struct mt_status_data status_data;
 987         char *xml_str = NULL;
 988         unsigned int i;
 989         int retval = 0;
 990
 991         retval = mt_get_xml_str(fd, MTIOCEXTGET, &xml_str);
 992         if (retval != 0)
 993                 err(1, "Couldn't get XML string from %s", filename);
 994
 995         retval = mt_get_status(xml_str, &status_data);
 996         if (retval != XML_STATUS_OK) {
 997                 warn("couldn't get status for %s", filename);
 998                 retval = 1;
 999                 goto bailout;
1000         } else
1001                 retval = 0;
1002
1003         if (status_data.error != 0) {
1004                 warnx("%s", status_data.error_str);
1005                 retval = 1;
1006                 goto bailout;
1007         }
1008
1009         for (i = 0; i < nitems(req_status_items); i++) {
1010                 char *name;
1011
1012                 name = __DECONST(char *, req_status_items[i].name);
1013                 req_status_items[i].entry = mt_status_entry_find(&status_data,
1014                     name);
1015                 if (req_status_items[i].entry == NULL) {
1016                         errx(1, "Cannot find status entry %s",
1017                             req_status_items[i].name);
1018                 }
1019         }
1020
1021         *max_iosize = req_status_items[CAMDD_TS_EFF_IOSIZE].entry->value_unsigned;
1022         *max_blk= req_status_items[CAMDD_TS_MAX_BLK].entry->value_unsigned;
1023         *min_blk= req_status_items[CAMDD_TS_MIN_BLK].entry->value_unsigned;
1024         *blk_gran = req_status_items[CAMDD_TS_BLK_GRAN].entry->value_unsigned;
1025 bailout:
1026
1027         free(xml_str);
1028         mt_status_free(&status_data);
1029
1030         return (retval);
1031 }
1032
1033 struct camdd_dev *
1034 camdd_probe_file(int fd, struct camdd_io_opts *io_opts, int retry_count,
1035     int timeout)
1036 {
1037         struct camdd_dev *dev = NULL;
1038         struct camdd_dev_file *file_dev;
1039         uint64_t blocksize = io_opts->blocksize;
1040
1041         dev = camdd_alloc_dev(CAMDD_DEV_FILE, NULL, 0, retry_count, timeout);
1042         if (dev == NULL)
1043                 goto bailout;
1044
1045         file_dev = &dev->dev_spec.file;
1046         file_dev->fd = fd;
1047         strlcpy(file_dev->filename, io_opts->dev_name,
1048             sizeof(file_dev->filename));
1049         strlcpy(dev->device_name, io_opts->dev_name, sizeof(dev->device_name));
1050         if (blocksize == 0)
1051                 dev->blocksize = CAMDD_FILE_DEFAULT_BLOCK;
1052         else
1053                 dev->blocksize = blocksize;
1054
1055         if ((io_opts->queue_depth != 0)
1056          && (io_opts->queue_depth != 1)) {
1057                 warnx("Queue depth %ju for %s ignored, only 1 outstanding "
1058                     "command supported", (uintmax_t)io_opts->queue_depth,
1059                     io_opts->dev_name);
1060         }
1061         dev->target_queue_depth = CAMDD_FILE_DEFAULT_DEPTH;
1062         dev->run = camdd_file_run;
1063         dev->fetch = NULL;
1064
1065         /*
1066          * We can effectively access files on byte boundaries.  We'll reset
1067          * this for devices like disks that can be accessed on sector
1068          * boundaries.
1069          */
1070         dev->sector_size = 1;
1071
1072         if ((fd != STDIN_FILENO)
1073          && (fd != STDOUT_FILENO)) {
1074                 int retval;
1075
1076                 retval = fstat(fd, &file_dev->sb);
1077                 if (retval != 0) {
1078                         warn("Cannot stat %s", dev->device_name);
1079                         goto bailout_error;
1080                 }
1081                 if (S_ISREG(file_dev->sb.st_mode)) {
1082                         file_dev->file_type = CAMDD_FILE_REG;
1083                 } else if (S_ISCHR(file_dev->sb.st_mode)) {
1084                         int type;
1085
1086                         if (ioctl(fd, FIODTYPE, &type) == -1)
1087                                 err(1, "FIODTYPE ioctl failed on %s",
1088                                     dev->device_name);
1089                         else {
1090                                 if (type & D_TAPE)
1091                                         file_dev->file_type = CAMDD_FILE_TAPE;
1092                                 else if (type & D_DISK)
1093                                         file_dev->file_type = CAMDD_FILE_DISK;
1094                                 else if (type & D_MEM)
1095                                         file_dev->file_type = CAMDD_FILE_MEM;
1096                                 else if (type & D_TTY)
1097                                         file_dev->file_type = CAMDD_FILE_TTY;
1098                         }
1099                 } else if (S_ISDIR(file_dev->sb.st_mode)) {
1100                         errx(1, "cannot operate on directory %s",
1101                             dev->device_name);
1102                 } else if (S_ISFIFO(file_dev->sb.st_mode)) {
1103                         file_dev->file_type = CAMDD_FILE_PIPE;
1104                 } else
1105                         errx(1, "Cannot determine file type for %s",
1106                             dev->device_name);
1107
1108                 switch (file_dev->file_type) {
1109                 case CAMDD_FILE_REG:
1110                         if (file_dev->sb.st_size != 0)
1111                                 dev->max_sector = file_dev->sb.st_size - 1;
1112                         else
1113                                 dev->max_sector = 0;
1114                         file_dev->file_flags |= CAMDD_FF_CAN_SEEK;
1115                         break;
1116                 case CAMDD_FILE_TAPE: {
1117                         uint64_t max_iosize, max_blk, min_blk, blk_gran;
1118                         /*
1119                          * Check block limits and maximum effective iosize.
1120                          * Make sure the blocksize is within the block
1121                          * limits (and a multiple of the minimum blocksize)
1122                          * and that the blocksize is <= maximum effective
1123                          * iosize.
1124                          */
1125                         retval = camdd_probe_tape(fd, dev->device_name,
1126                             &max_iosize, &max_blk, &min_blk, &blk_gran);
1127                         if (retval != 0)
1128                                 errx(1, "Unable to probe tape %s",
1129                                     dev->device_name);
1130
1131                         /*
1132                          * The blocksize needs to be <= the maximum
1133                          * effective I/O size of the tape device.  Note
1134                          * that this also takes into account the maximum
1135                          * blocksize reported by READ BLOCK LIMITS.
1136                          */
1137                         if (dev->blocksize > max_iosize) {
1138                                 warnx("Blocksize %u too big for %s, limiting "
1139                                     "to %ju", dev->blocksize, dev->device_name,
1140                                     max_iosize);
1141                                 dev->blocksize = max_iosize;
1142                         }
1143
1144                         /*
1145                          * The blocksize needs to be at least min_blk;
1146                          */
1147                         if (dev->blocksize < min_blk) {
1148                                 warnx("Blocksize %u too small for %s, "
1149                                     "increasing to %ju", dev->blocksize,
1150                                     dev->device_name, min_blk);
1151                                 dev->blocksize = min_blk;
1152                         }
1153
1154                         /*
1155                          * And the blocksize needs to be a multiple of
1156                          * the block granularity.
1157                          */
1158                         if ((blk_gran != 0)
1159                          && (dev->blocksize % (1 << blk_gran))) {
1160                                 warnx("Blocksize %u for %s not a multiple of "
1161                                     "%d, adjusting to %d", dev->blocksize,
1162                                     dev->device_name, (1 << blk_gran),
1163                                     dev->blocksize & ~((1 << blk_gran) - 1));
1164                                 dev->blocksize &= ~((1 << blk_gran) - 1);
1165                         }
1166
1167                         if (dev->blocksize == 0) {
1168                                 errx(1, "Unable to derive valid blocksize for "
1169                                     "%s", dev->device_name);
1170                         }
1171
1172                         /*
1173                          * For tape drives, set the sector size to the
1174                          * blocksize so that we make sure not to write
1175                          * less than the blocksize out to the drive.
1176                          */
1177                         dev->sector_size = dev->blocksize;
1178                         break;
1179                 }
1180                 case CAMDD_FILE_DISK: {
1181                         off_t media_size;
1182                         unsigned int sector_size;
1183
1184                         file_dev->file_flags |= CAMDD_FF_CAN_SEEK;
1185
1186                         if (ioctl(fd, DIOCGSECTORSIZE, &sector_size) == -1) {
1187                                 err(1, "DIOCGSECTORSIZE ioctl failed on %s",
1188                                     dev->device_name);
1189                         }
1190
1191                         if (sector_size == 0) {
1192                                 errx(1, "DIOCGSECTORSIZE ioctl returned "
1193                                     "invalid sector size %u for %s",
1194                                     sector_size, dev->device_name);
1195                         }
1196
1197                         if (ioctl(fd, DIOCGMEDIASIZE, &media_size) == -1) {
1198                                 err(1, "DIOCGMEDIASIZE ioctl failed on %s",
1199                                     dev->device_name);
1200                         }
1201
1202                         if (media_size == 0) {
1203                                 errx(1, "DIOCGMEDIASIZE ioctl returned "
1204                                     "invalid media size %ju for %s",
1205                                     (uintmax_t)media_size, dev->device_name);
1206                         }
1207
1208                         if (dev->blocksize % sector_size) {
1209                                 errx(1, "%s blocksize %u not a multiple of "
1210                                     "sector size %u", dev->device_name,
1211                                     dev->blocksize, sector_size);
1212                         }
1213
1214                         dev->sector_size = sector_size;
1215                         dev->max_sector = (media_size / sector_size) - 1;
1216                         break;
1217                 }
1218                 case CAMDD_FILE_MEM:
1219                         file_dev->file_flags |= CAMDD_FF_CAN_SEEK;
1220                         break;
1221                 default:
1222                         break;
1223                 }
1224         }
1225
1226         if ((io_opts->offset != 0)
1227          && ((file_dev->file_flags & CAMDD_FF_CAN_SEEK) == 0)) {
1228                 warnx("Offset %ju specified for %s, but we cannot seek on %s",
1229                     io_opts->offset, io_opts->dev_name, io_opts->dev_name);
1230                 goto bailout_error;
1231         }
1232 #if 0
1233         else if ((io_opts->offset != 0)
1234                 && ((io_opts->offset % dev->sector_size) != 0)) {
1235                 warnx("Offset %ju for %s is not a multiple of the "
1236                       "sector size %u", io_opts->offset,
1237                       io_opts->dev_name, dev->sector_size);
1238                 goto bailout_error;
1239         } else {
1240                 dev->start_offset_bytes = io_opts->offset;
1241         }
1242 #endif
1243
1244 bailout:
1245         return (dev);
1246
1247 bailout_error:
1248         camdd_free_dev(dev);
1249         return (NULL);
1250 }
1251
1252 /*
1253  * Get a get device CCB for the specified device.
1254  */
1255 int
1256 camdd_get_cgd(struct cam_device *device, struct ccb_getdev *cgd)
1257 {
1258         union ccb *ccb;
1259         int retval = 0;
1260
1261         ccb = cam_getccb(device);
1262
1263         if (ccb == NULL) {
1264                 warnx("%s: couldn't allocate CCB", __func__);
1265                 return -1;
1266         }
1267
1268         CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->cgd);
1269
1270         ccb->ccb_h.func_code = XPT_GDEV_TYPE;
1271
1272         if (cam_send_ccb(device, ccb) < 0) {
1273                 warn("%s: error sending Get Device Information CCB", __func__);
1274                         cam_error_print(device, ccb, CAM_ESF_ALL,
1275                                         CAM_EPF_ALL, stderr);
1276                 retval = -1;
1277                 goto bailout;
1278         }
1279
1280         if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1281                         cam_error_print(device, ccb, CAM_ESF_ALL,
1282                                         CAM_EPF_ALL, stderr);
1283                 retval = -1;
1284                 goto bailout;
1285         }
1286
1287         bcopy(&ccb->cgd, cgd, sizeof(struct ccb_getdev));
1288
1289 bailout:
1290         cam_freeccb(ccb);
1291
1292         return retval;
1293 }
1294
1295 int
1296 camdd_probe_pass_scsi(struct cam_device *cam_dev, union ccb *ccb,
1297                  camdd_argmask arglist, int probe_retry_count,
1298                  int probe_timeout, uint64_t *maxsector, uint32_t *block_len)
1299 {
1300         struct scsi_read_capacity_data rcap;
1301         struct scsi_read_capacity_data_long rcaplong;
1302         int retval = -1;
1303
1304         if (ccb == NULL) {
1305                 warnx("%s: error passed ccb is NULL", __func__);
1306                 goto bailout;
1307         }
1308
1309         CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->csio);
1310
1311         scsi_read_capacity(&ccb->csio,
1312                            /*retries*/ probe_retry_count,
1313                            /*cbfcnp*/ NULL,
1314                            /*tag_action*/ MSG_SIMPLE_Q_TAG,
1315                            &rcap,
1316                            SSD_FULL_SIZE,
1317                            /*timeout*/ probe_timeout ? probe_timeout : 5000);
1318
1319         /* Disable freezing the device queue */
1320         ccb->ccb_h.flags |= CAM_DEV_QFRZDIS;
1321
1322         if (arglist & CAMDD_ARG_ERR_RECOVER)
1323                 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
1324
1325         if (cam_send_ccb(cam_dev, ccb) < 0) {
1326                 warn("error sending READ CAPACITY command");
1327
1328                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL,
1329                                 CAM_EPF_ALL, stderr);
1330
1331                 goto bailout;
1332         }
1333
1334         if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1335                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr);
1336                 goto bailout;
1337         }
1338
1339         *maxsector = scsi_4btoul(rcap.addr);
1340         *block_len = scsi_4btoul(rcap.length);
1341
1342         /*
1343          * A last block of 2^32-1 means that the true capacity is over 2TB,
1344          * and we need to issue the long READ CAPACITY to get the real
1345          * capacity.  Otherwise, we're all set.
1346          */
1347         if (*maxsector != 0xffffffff) {
1348                 retval = 0;
1349                 goto bailout;
1350         }
1351
1352         scsi_read_capacity_16(&ccb->csio,
1353                               /*retries*/ probe_retry_count,
1354                               /*cbfcnp*/ NULL,
1355                               /*tag_action*/ MSG_SIMPLE_Q_TAG,
1356                               /*lba*/ 0,
1357                               /*reladdr*/ 0,
1358                               /*pmi*/ 0,
1359                               (uint8_t *)&rcaplong,
1360                               sizeof(rcaplong),
1361                               /*sense_len*/ SSD_FULL_SIZE,
1362                               /*timeout*/ probe_timeout ? probe_timeout : 5000);
1363
1364         /* Disable freezing the device queue */
1365         ccb->ccb_h.flags |= CAM_DEV_QFRZDIS;
1366
1367         if (arglist & CAMDD_ARG_ERR_RECOVER)
1368                 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
1369
1370         if (cam_send_ccb(cam_dev, ccb) < 0) {
1371                 warn("error sending READ CAPACITY (16) command");
1372                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL,
1373                                 CAM_EPF_ALL, stderr);
1374                 goto bailout;
1375         }
1376
1377         if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1378                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr);
1379                 goto bailout;
1380         }
1381
1382         *maxsector = scsi_8btou64(rcaplong.addr);
1383         *block_len = scsi_4btoul(rcaplong.length);
1384
1385         retval = 0;
1386
1387 bailout:
1388         return retval;
1389 }
1390
1391 int
1392 camdd_probe_pass_nvme(struct cam_device *cam_dev, union ccb *ccb,
1393                  camdd_argmask arglist, int probe_retry_count,
1394                  int probe_timeout, uint64_t *maxsector, uint32_t *block_len)
1395 {
1396         struct nvme_command *nc = NULL;
1397         struct nvme_namespace_data nsdata;
1398         uint32_t nsid = cam_dev->target_lun & UINT32_MAX;
1399         uint8_t format = 0, lbads = 0;
1400         int retval = -1;
1401
1402         if (ccb == NULL) {
1403                 warnx("%s: error passed ccb is NULL", __func__);
1404                 goto bailout;
1405         }
1406
1407         CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->nvmeio);
1408
1409         /* Send Identify Namespace to get block size and capacity */
1410         nc = &ccb->nvmeio.cmd;
1411         nc->opc = NVME_OPC_IDENTIFY;
1412
1413         nc->nsid = nsid;
1414         nc->cdw10 = 0; /* Identify Namespace is CNS = 0 */
1415
1416         cam_fill_nvmeadmin(&ccb->nvmeio,
1417                         /*retries*/ probe_retry_count,
1418                         /*cbfcnp*/ NULL,
1419                         CAM_DIR_IN,
1420                         (uint8_t *)&nsdata,
1421                         sizeof(nsdata),
1422                         probe_timeout);
1423
1424         /* Disable freezing the device queue */
1425         ccb->ccb_h.flags |= CAM_DEV_QFRZDIS;
1426
1427         if (arglist & CAMDD_ARG_ERR_RECOVER)
1428                 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
1429
1430         if (cam_send_ccb(cam_dev, ccb) < 0) {
1431                 warn("error sending Identify Namespace command");
1432
1433                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL,
1434                                 CAM_EPF_ALL, stderr);
1435
1436                 goto bailout;
1437         }
1438
1439         if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1440                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr);
1441                 goto bailout;
1442         }
1443
1444         *maxsector = nsdata.nsze;
1445         /* The LBA Data Size (LBADS) is reported as a power of 2 */
1446         format = nsdata.flbas & NVME_NS_DATA_FLBAS_FORMAT_MASK;
1447         lbads = (nsdata.lbaf[format] >> NVME_NS_DATA_LBAF_LBADS_SHIFT) &
1448             NVME_NS_DATA_LBAF_LBADS_MASK;
1449         *block_len = 1 << lbads;
1450
1451         retval = 0;
1452
1453 bailout:
1454         return retval;
1455 }
1456
1457 /*
1458  * Need to implement this.  Do a basic probe:
1459  * - Check the inquiry data, make sure we're talking to a device that we
1460  *   can reasonably expect to talk to -- direct, RBC, CD, WORM.
1461  * - Send a test unit ready, make sure the device is available.
1462  * - Get the capacity and block size.
1463  */
1464 struct camdd_dev *
1465 camdd_probe_pass(struct cam_device *cam_dev, struct camdd_io_opts *io_opts,
1466                  camdd_argmask arglist, int probe_retry_count,
1467                  int probe_timeout, int io_retry_count, int io_timeout)
1468 {
1469         union ccb *ccb;
1470         uint64_t maxsector = 0;
1471         uint32_t cpi_maxio, max_iosize, pass_numblocks;
1472         uint32_t block_len = 0;
1473         struct camdd_dev *dev = NULL;
1474         struct camdd_dev_pass *pass_dev;
1475         struct kevent ke;
1476         struct ccb_getdev cgd;
1477         int retval;
1478         int scsi_dev_type = T_NODEVICE;
1479
1480         if ((retval = camdd_get_cgd(cam_dev, &cgd)) != 0) {
1481                 warnx("%s: error retrieving CGD", __func__);
1482                 return NULL;
1483         }
1484
1485         ccb = cam_getccb(cam_dev);
1486
1487         if (ccb == NULL) {
1488                 warnx("%s: error allocating ccb", __func__);
1489                 goto bailout;
1490         }
1491
1492         switch (cgd.protocol) {
1493         case PROTO_SCSI:
1494                 scsi_dev_type = SID_TYPE(&cam_dev->inq_data);
1495
1496                 /*
1497                  * For devices that support READ CAPACITY, we'll attempt to get the
1498                  * capacity.  Otherwise, we really don't support tape or other
1499                  * devices via SCSI passthrough, so just return an error in that case.
1500                  */
1501                 switch (scsi_dev_type) {
1502                 case T_DIRECT:
1503                 case T_WORM:
1504                 case T_CDROM:
1505                 case T_OPTICAL:
1506                 case T_RBC:
1507                 case T_ZBC_HM:
1508                         break;
1509                 default:
1510                         errx(1, "Unsupported SCSI device type %d", scsi_dev_type);
1511                         break; /*NOTREACHED*/
1512                 }
1513
1514                 if ((retval = camdd_probe_pass_scsi(cam_dev, ccb, probe_retry_count,
1515                                                 arglist, probe_timeout, &maxsector,
1516                                                 &block_len))) {
1517                         goto bailout;
1518                 }
1519                 break;
1520         case PROTO_NVME:
1521                 if ((retval = camdd_probe_pass_nvme(cam_dev, ccb, probe_retry_count,
1522                                                 arglist, probe_timeout, &maxsector,
1523                                                 &block_len))) {
1524                         goto bailout;
1525                 }
1526                 break;
1527         default:
1528                 errx(1, "Unsupported PROTO type %d", cgd.protocol);
1529                 break; /*NOTREACHED*/
1530         }
1531
1532         if (block_len == 0) {
1533                 warnx("Sector size for %s%u is 0, cannot continue",
1534                     cam_dev->device_name, cam_dev->dev_unit_num);
1535                 goto bailout_error;
1536         }
1537
1538         CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->cpi);
1539
1540         ccb->ccb_h.func_code = XPT_PATH_INQ;
1541         ccb->ccb_h.flags = CAM_DIR_NONE;
1542         ccb->ccb_h.retry_count = 1;
1543
1544         if (cam_send_ccb(cam_dev, ccb) < 0) {
1545                 warn("error sending XPT_PATH_INQ CCB");
1546
1547                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL,
1548                                 CAM_EPF_ALL, stderr);
1549                 goto bailout;
1550         }
1551
1552         EV_SET(&ke, cam_dev->fd, EVFILT_READ, EV_ADD|EV_ENABLE, 0, 0, 0);
1553
1554         dev = camdd_alloc_dev(CAMDD_DEV_PASS, &ke, 1, io_retry_count,
1555                               io_timeout);
1556         if (dev == NULL)
1557                 goto bailout;
1558
1559         pass_dev = &dev->dev_spec.pass;
1560         pass_dev->scsi_dev_type = scsi_dev_type;
1561         pass_dev->protocol = cgd.protocol;
1562         pass_dev->dev = cam_dev;
1563         pass_dev->max_sector = maxsector;
1564         pass_dev->block_len = block_len;
1565         pass_dev->cpi_maxio = ccb->cpi.maxio;
1566         snprintf(dev->device_name, sizeof(dev->device_name), "%s%u",
1567                  pass_dev->dev->device_name, pass_dev->dev->dev_unit_num);
1568         dev->sector_size = block_len;
1569         dev->max_sector = maxsector;
1570
1571
1572         /*
1573          * Determine the optimal blocksize to use for this device.
1574          */
1575
1576         /*
1577          * If the controller has not specified a maximum I/O size,
1578          * just go with 128K as a somewhat conservative value.
1579          */
1580         if (pass_dev->cpi_maxio == 0)
1581                 cpi_maxio = 131072;
1582         else
1583                 cpi_maxio = pass_dev->cpi_maxio;
1584
1585         /*
1586          * If the controller has a large maximum I/O size, limit it
1587          * to something smaller so that the kernel doesn't have trouble
1588          * allocating buffers to copy data in and out for us.
1589          * XXX KDM this is until we have unmapped I/O support in the kernel.
1590          */
1591         max_iosize = min(cpi_maxio, CAMDD_PASS_MAX_BLOCK);
1592
1593         /*
1594          * If we weren't able to get a block size for some reason,
1595          * default to 512 bytes.
1596          */
1597         block_len = pass_dev->block_len;
1598         if (block_len == 0)
1599                 block_len = 512;
1600
1601         /*
1602          * Figure out how many blocksize chunks will fit in the
1603          * maximum I/O size.
1604          */
1605         pass_numblocks = max_iosize / block_len;
1606
1607         /*
1608          * And finally, multiple the number of blocks by the LBA
1609          * length to get our maximum block size;
1610          */
1611         dev->blocksize = pass_numblocks * block_len;
1612
1613         if (io_opts->blocksize != 0) {
1614                 if ((io_opts->blocksize % dev->sector_size) != 0) {
1615                         warnx("Blocksize %ju for %s is not a multiple of "
1616                               "sector size %u", (uintmax_t)io_opts->blocksize,
1617                               dev->device_name, dev->sector_size);
1618                         goto bailout_error;
1619                 }
1620                 dev->blocksize = io_opts->blocksize;
1621         }
1622         dev->target_queue_depth = CAMDD_PASS_DEFAULT_DEPTH;
1623         if (io_opts->queue_depth != 0)
1624                 dev->target_queue_depth = io_opts->queue_depth;
1625
1626         if (io_opts->offset != 0) {
1627                 if (io_opts->offset > (dev->max_sector * dev->sector_size)) {
1628                         warnx("Offset %ju is past the end of device %s",
1629                             io_opts->offset, dev->device_name);
1630                         goto bailout_error;
1631                 }
1632 #if 0
1633                 else if ((io_opts->offset % dev->sector_size) != 0) {
1634                         warnx("Offset %ju for %s is not a multiple of the "
1635                               "sector size %u", io_opts->offset,
1636                               dev->device_name, dev->sector_size);
1637                         goto bailout_error;
1638                 }
1639                 dev->start_offset_bytes = io_opts->offset;
1640 #endif
1641         }
1642
1643         dev->min_cmd_size = io_opts->min_cmd_size;
1644
1645         dev->run = camdd_pass_run;
1646         dev->fetch = camdd_pass_fetch;
1647
1648 bailout:
1649         cam_freeccb(ccb);
1650
1651         return (dev);
1652
1653 bailout_error:
1654         cam_freeccb(ccb);
1655
1656         camdd_free_dev(dev);
1657
1658         return (NULL);
1659 }
1660
1661 void
1662 nvme_read_write(struct ccb_nvmeio *nvmeio, uint32_t retries,
1663                 void (*cbfcnp)(struct cam_periph *, union ccb *),
1664                 uint32_t nsid, int readop, uint64_t lba,
1665                 uint32_t block_count, uint8_t *data_ptr, uint32_t dxfer_len,
1666                 uint32_t timeout)
1667 {
1668         struct nvme_command *nc = &nvmeio->cmd;
1669
1670         nc->opc = readop ? NVME_OPC_READ : NVME_OPC_WRITE;
1671
1672         nc->nsid = nsid;
1673
1674         nc->cdw10 = lba & UINT32_MAX;
1675         nc->cdw11 = lba >> 32;
1676
1677         /* NLB (bits 15:0) is a zero based value */
1678         nc->cdw12 = (block_count - 1) & UINT16_MAX;
1679
1680         cam_fill_nvmeio(nvmeio,
1681                         retries,
1682                         cbfcnp,
1683                         readop ? CAM_DIR_IN : CAM_DIR_OUT,
1684                         data_ptr,
1685                         dxfer_len,
1686                         timeout);
1687 }
1688
1689 void *
1690 camdd_worker(void *arg)
1691 {
1692         struct camdd_dev *dev = arg;
1693         struct camdd_buf *buf;
1694         struct timespec ts, *kq_ts;
1695
1696         ts.tv_sec = 0;
1697         ts.tv_nsec = 0;
1698
1699         pthread_mutex_lock(&dev->mutex);
1700
1701         dev->flags |= CAMDD_DEV_FLAG_ACTIVE;
1702
1703         for (;;) {
1704                 struct kevent ke;
1705                 int retval = 0;
1706
1707                 /*
1708                  * XXX KDM check the reorder queue depth?
1709                  */
1710                 if (dev->write_dev == 0) {
1711                         uint32_t our_depth, peer_depth, peer_bytes, our_bytes;
1712                         uint32_t target_depth = dev->target_queue_depth;
1713                         uint32_t peer_target_depth =
1714                             dev->peer_dev->target_queue_depth;
1715                         uint32_t peer_blocksize = dev->peer_dev->blocksize;
1716
1717                         camdd_get_depth(dev, &our_depth, &peer_depth,
1718                                         &our_bytes, &peer_bytes);
1719
1720 #if 0
1721                         while (((our_depth < target_depth)
1722                              && (peer_depth < peer_target_depth))
1723                             || ((peer_bytes + our_bytes) <
1724                                  (peer_blocksize * 2))) {
1725 #endif
1726                         while (((our_depth + peer_depth) <
1727                                 (target_depth + peer_target_depth))
1728                             || ((peer_bytes + our_bytes) <
1729                                 (peer_blocksize * 3))) {
1730
1731                                 retval = camdd_queue(dev, NULL);
1732                                 if (retval == 1)
1733                                         break;
1734                                 else if (retval != 0) {
1735                                         error_exit = 1;
1736                                         goto bailout;
1737                                 }
1738
1739                                 camdd_get_depth(dev, &our_depth, &peer_depth,
1740                                                 &our_bytes, &peer_bytes);
1741                         }
1742                 }
1743                 /*
1744                  * See if we have any I/O that is ready to execute.
1745                  */
1746                 buf = STAILQ_FIRST(&dev->run_queue);
1747                 if (buf != NULL) {
1748                         while (dev->target_queue_depth > dev->cur_active_io) {
1749                                 retval = dev->run(dev);
1750                                 if (retval == -1) {
1751                                         dev->flags |= CAMDD_DEV_FLAG_EOF;
1752                                         error_exit = 1;
1753                                         break;
1754                                 } else if (retval != 0) {
1755                                         break;
1756                                 }
1757                         }
1758                 }
1759
1760                 /*
1761                  * We've reached EOF, or our partner has reached EOF.
1762                  */
1763                 if ((dev->flags & CAMDD_DEV_FLAG_EOF)
1764                  || (dev->flags & CAMDD_DEV_FLAG_PEER_EOF)) {
1765                         if (dev->write_dev != 0) {
1766                                 if ((STAILQ_EMPTY(&dev->work_queue))
1767                                  && (dev->num_run_queue == 0)
1768                                  && (dev->cur_active_io == 0)) {
1769                                         goto bailout;
1770                                 }
1771                         } else {
1772                                 /*
1773                                  * If we're the reader, and the writer
1774                                  * got EOF, he is already done.  If we got
1775                                  * the EOF, then we need to wait until
1776                                  * everything is flushed out for the writer.
1777                                  */
1778                                 if (dev->flags & CAMDD_DEV_FLAG_PEER_EOF) {
1779                                         goto bailout;
1780                                 } else if ((dev->num_peer_work_queue == 0)
1781                                         && (dev->num_peer_done_queue == 0)
1782                                         && (dev->cur_active_io == 0)
1783                                         && (dev->num_run_queue == 0)) {
1784                                         goto bailout;
1785                                 }
1786                         }
1787                         /*
1788                          * XXX KDM need to do something about the pending
1789                          * queue and cleanup resources.
1790                          */
1791                 }
1792
1793                 if ((dev->write_dev == 0)
1794                  && (dev->cur_active_io == 0)
1795                  && (dev->peer_bytes_queued < dev->peer_dev->blocksize))
1796                         kq_ts = &ts;
1797                 else
1798                         kq_ts = NULL;
1799
1800                 /*
1801                  * Run kevent to see if there are events to process.
1802                  */
1803                 pthread_mutex_unlock(&dev->mutex);
1804                 retval = kevent(dev->kq, NULL, 0, &ke, 1, kq_ts);
1805                 pthread_mutex_lock(&dev->mutex);
1806                 if (retval == -1) {
1807                         warn("%s: error returned from kevent",__func__);
1808                         goto bailout;
1809                 } else if (retval != 0) {
1810                         switch (ke.filter) {
1811                         case EVFILT_READ:
1812                                 if (dev->fetch != NULL) {
1813                                         retval = dev->fetch(dev);
1814                                         if (retval == -1) {
1815                                                 error_exit = 1;
1816                                                 goto bailout;
1817                                         }
1818                                 }
1819                                 break;
1820                         case EVFILT_SIGNAL:
1821                                 /*
1822                                  * We register for this so we don't get
1823                                  * an error as a result of a SIGINFO or a
1824                                  * SIGINT.  It will actually get handled
1825                                  * by the signal handler.  If we get a
1826                                  * SIGINT, bail out without printing an
1827                                  * error message.  Any other signals
1828                                  * will result in the error message above.
1829                                  */
1830                                 if (ke.ident == SIGINT)
1831                                         goto bailout;
1832                                 break;
1833                         case EVFILT_USER:
1834                                 retval = 0;
1835                                 /*
1836                                  * Check to see if the other thread has
1837                                  * queued any I/O for us to do.  (In this
1838                                  * case we're the writer.)
1839                                  */
1840                                 for (buf = STAILQ_FIRST(&dev->work_queue);
1841                                      buf != NULL;
1842                                      buf = STAILQ_FIRST(&dev->work_queue)) {
1843                                         STAILQ_REMOVE_HEAD(&dev->work_queue,
1844                                                            work_links);
1845                                         retval = camdd_queue(dev, buf);
1846                                         /*
1847                                          * We keep going unless we get an
1848                                          * actual error.  If we get EOF, we
1849                                          * still want to remove the buffers
1850                                          * from the queue and send the back
1851                                          * to the reader thread.
1852                                          */
1853                                         if (retval == -1) {
1854                                                 error_exit = 1;
1855                                                 goto bailout;
1856                                         } else
1857                                                 retval = 0;
1858                                 }
1859
1860                                 /*
1861                                  * Next check to see if the other thread has
1862                                  * queued any completed buffers back to us.
1863                                  * (In this case we're the reader.)
1864                                  */
1865                                 for (buf = STAILQ_FIRST(&dev->peer_done_queue);
1866                                      buf != NULL;
1867                                      buf = STAILQ_FIRST(&dev->peer_done_queue)){
1868                                         STAILQ_REMOVE_HEAD(
1869                                             &dev->peer_done_queue, work_links);
1870                                         dev->num_peer_done_queue--;
1871                                         camdd_peer_done(buf);
1872                                 }
1873                                 break;
1874                         default:
1875                                 warnx("%s: unknown kevent filter %d",
1876                                       __func__, ke.filter);
1877                                 break;
1878                         }
1879                 }
1880         }
1881
1882 bailout:
1883
1884         dev->flags &= ~CAMDD_DEV_FLAG_ACTIVE;
1885
1886         /* XXX KDM cleanup resources here? */
1887
1888         pthread_mutex_unlock(&dev->mutex);
1889
1890         need_exit = 1;
1891         sem_post(&camdd_sem);
1892
1893         return (NULL);
1894 }
1895
1896 /*
1897  * Simplistic translation of CCB status to our local status.
1898  */
1899 camdd_buf_status
1900 camdd_ccb_status(union ccb *ccb, int protocol)
1901 {
1902         camdd_buf_status status = CAMDD_STATUS_NONE;
1903         cam_status ccb_status;
1904
1905         ccb_status = ccb->ccb_h.status & CAM_STATUS_MASK;
1906
1907         switch (protocol) {
1908         case PROTO_SCSI:
1909                 switch (ccb_status) {
1910                 case CAM_REQ_CMP: {
1911                         if (ccb->csio.resid == 0) {
1912                                 status = CAMDD_STATUS_OK;
1913                         } else if (ccb->csio.dxfer_len > ccb->csio.resid) {
1914                                 status = CAMDD_STATUS_SHORT_IO;
1915                         } else {
1916                                 status = CAMDD_STATUS_EOF;
1917                         }
1918                         break;
1919                 }
1920                 case CAM_SCSI_STATUS_ERROR: {
1921                         switch (ccb->csio.scsi_status) {
1922                         case SCSI_STATUS_OK:
1923                         case SCSI_STATUS_COND_MET:
1924                         case SCSI_STATUS_INTERMED:
1925                         case SCSI_STATUS_INTERMED_COND_MET:
1926                                 status = CAMDD_STATUS_OK;
1927                                 break;
1928                         case SCSI_STATUS_CMD_TERMINATED:
1929                         case SCSI_STATUS_CHECK_COND:
1930                         case SCSI_STATUS_QUEUE_FULL:
1931                         case SCSI_STATUS_BUSY:
1932                         case SCSI_STATUS_RESERV_CONFLICT:
1933                         default:
1934                                 status = CAMDD_STATUS_ERROR;
1935                                 break;
1936                         }
1937                         break;
1938                 }
1939                 default:
1940                         status = CAMDD_STATUS_ERROR;
1941                         break;
1942                 }
1943                 break;
1944         case PROTO_NVME:
1945                 switch (ccb_status) {
1946                 case CAM_REQ_CMP:
1947                         status = CAMDD_STATUS_OK;
1948                         break;
1949                 default:
1950                         status = CAMDD_STATUS_ERROR;
1951                         break;
1952                 }
1953                 break;
1954         default:
1955                 status = CAMDD_STATUS_ERROR;
1956                 break;
1957         }
1958
1959         return (status);
1960 }
1961
1962 /*
1963  * Queue a buffer to our peer's work thread for writing.
1964  *
1965  * Returns 0 for success, -1 for failure, 1 if the other thread exited.
1966  */
1967 int
1968 camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf)
1969 {
1970         struct kevent ke;
1971         STAILQ_HEAD(, camdd_buf) local_queue;
1972         struct camdd_buf *buf1, *buf2;
1973         struct camdd_buf_data *data = NULL;
1974         uint64_t peer_bytes_queued = 0;
1975         int active = 1;
1976         int retval = 0;
1977
1978         STAILQ_INIT(&local_queue);
1979
1980         /*
1981          * Since we're the reader, we need to queue our I/O to the writer
1982          * in sequential order in order to make sure it gets written out
1983          * in sequential order.
1984          *
1985          * Check the next expected I/O starting offset.  If this doesn't
1986          * match, put it on the reorder queue.
1987          */
1988         if ((buf->lba * dev->sector_size) != dev->next_completion_pos_bytes) {
1989
1990                 /*
1991                  * If there is nothing on the queue, there is no sorting
1992                  * needed.
1993                  */
1994                 if (STAILQ_EMPTY(&dev->reorder_queue)) {
1995                         STAILQ_INSERT_TAIL(&dev->reorder_queue, buf, links);
1996                         dev->num_reorder_queue++;
1997                         goto bailout;
1998                 }
1999
2000                 /*
2001                  * Sort in ascending order by starting LBA.  There should
2002                  * be no identical LBAs.
2003                  */
2004                 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL;
2005                      buf1 = buf2) {
2006                         buf2 = STAILQ_NEXT(buf1, links);
2007                         if (buf->lba < buf1->lba) {
2008                                 /*
2009                                  * If we're less than the first one, then
2010                                  * we insert at the head of the list
2011                                  * because this has to be the first element
2012                                  * on the list.
2013                                  */
2014                                 STAILQ_INSERT_HEAD(&dev->reorder_queue,
2015                                                    buf, links);
2016                                 dev->num_reorder_queue++;
2017                                 break;
2018                         } else if (buf->lba > buf1->lba) {
2019                                 if (buf2 == NULL) {
2020                                         STAILQ_INSERT_TAIL(&dev->reorder_queue,
2021                                             buf, links);
2022                                         dev->num_reorder_queue++;
2023                                         break;
2024                                 } else if (buf->lba < buf2->lba) {
2025                                         STAILQ_INSERT_AFTER(&dev->reorder_queue,
2026                                             buf1, buf, links);
2027                                         dev->num_reorder_queue++;
2028                                         break;
2029                                 }
2030                         } else {
2031                                 errx(1, "Found buffers with duplicate LBA %ju!",
2032                                      buf->lba);
2033                         }
2034                 }
2035                 goto bailout;
2036         } else {
2037
2038                 /*
2039                  * We're the next expected I/O completion, so put ourselves
2040                  * on the local queue to be sent to the writer.  We use
2041                  * work_links here so that we can queue this to the
2042                  * peer_work_queue before taking the buffer off of the
2043                  * local_queue.
2044                  */
2045                 dev->next_completion_pos_bytes += buf->len;
2046                 STAILQ_INSERT_TAIL(&local_queue, buf, work_links);
2047
2048                 /*
2049                  * Go through the reorder queue looking for more sequential
2050                  * I/O and add it to the local queue.
2051                  */
2052                 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL;
2053                      buf1 = STAILQ_FIRST(&dev->reorder_queue)) {
2054                         /*
2055                          * As soon as we see an I/O that is out of sequence,
2056                          * we're done.
2057                          */
2058                         if ((buf1->lba * dev->sector_size) !=
2059                              dev->next_completion_pos_bytes)
2060                                 break;
2061
2062                         STAILQ_REMOVE_HEAD(&dev->reorder_queue, links);
2063                         dev->num_reorder_queue--;
2064                         STAILQ_INSERT_TAIL(&local_queue, buf1, work_links);
2065                         dev->next_completion_pos_bytes += buf1->len;
2066                 }
2067         }
2068
2069         /*
2070          * Setup the event to let the other thread know that it has work
2071          * pending.
2072          */
2073         EV_SET(&ke, (uintptr_t)&dev->peer_dev->work_queue, EVFILT_USER, 0,
2074                NOTE_TRIGGER, 0, NULL);
2075
2076         /*
2077          * Put this on our shadow queue so that we know what we've queued
2078          * to the other thread.
2079          */
2080         STAILQ_FOREACH_SAFE(buf1, &local_queue, work_links, buf2) {
2081                 if (buf1->buf_type != CAMDD_BUF_DATA) {
2082                         errx(1, "%s: should have a data buffer, not an "
2083                             "indirect buffer", __func__);
2084                 }
2085                 data = &buf1->buf_type_spec.data;
2086
2087                 /*
2088                  * We only need to send one EOF to the writer, and don't
2089                  * need to continue sending EOFs after that.
2090                  */
2091                 if (buf1->status == CAMDD_STATUS_EOF) {
2092                         if (dev->flags & CAMDD_DEV_FLAG_EOF_SENT) {
2093                                 STAILQ_REMOVE(&local_queue, buf1, camdd_buf,
2094                                     work_links);
2095                                 camdd_release_buf(buf1);
2096                                 retval = 1;
2097                                 continue;
2098                         }
2099                         dev->flags |= CAMDD_DEV_FLAG_EOF_SENT;
2100                 }
2101
2102
2103                 STAILQ_INSERT_TAIL(&dev->peer_work_queue, buf1, links);
2104                 peer_bytes_queued += (data->fill_len - data->resid);
2105                 dev->peer_bytes_queued += (data->fill_len - data->resid);
2106                 dev->num_peer_work_queue++;
2107         }
2108
2109         if (STAILQ_FIRST(&local_queue) == NULL)
2110                 goto bailout;
2111
2112         /*
2113          * Drop our mutex and pick up the other thread's mutex.  We need to
2114          * do this to avoid deadlocks.
2115          */
2116         pthread_mutex_unlock(&dev->mutex);
2117         pthread_mutex_lock(&dev->peer_dev->mutex);
2118
2119         if (dev->peer_dev->flags & CAMDD_DEV_FLAG_ACTIVE) {
2120                 /*
2121                  * Put the buffers on the other thread's incoming work queue.
2122                  */
2123                 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL;
2124                      buf1 = STAILQ_FIRST(&local_queue)) {
2125                         STAILQ_REMOVE_HEAD(&local_queue, work_links);
2126                         STAILQ_INSERT_TAIL(&dev->peer_dev->work_queue, buf1,
2127                                            work_links);
2128                 }
2129                 /*
2130                  * Send an event to the other thread's kqueue to let it know
2131                  * that there is something on the work queue.
2132                  */
2133                 retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL);
2134                 if (retval == -1)
2135                         warn("%s: unable to add peer work_queue kevent",
2136                              __func__);
2137                 else
2138                         retval = 0;
2139         } else
2140                 active = 0;
2141
2142         pthread_mutex_unlock(&dev->peer_dev->mutex);
2143         pthread_mutex_lock(&dev->mutex);
2144
2145         /*
2146          * If the other side isn't active, run through the queue and
2147          * release all of the buffers.
2148          */
2149         if (active == 0) {
2150                 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL;
2151                      buf1 = STAILQ_FIRST(&local_queue)) {
2152                         STAILQ_REMOVE_HEAD(&local_queue, work_links);
2153                         STAILQ_REMOVE(&dev->peer_work_queue, buf1, camdd_buf,
2154                                       links);
2155                         dev->num_peer_work_queue--;
2156                         camdd_release_buf(buf1);
2157                 }
2158                 dev->peer_bytes_queued -= peer_bytes_queued;
2159                 retval = 1;
2160         }
2161
2162 bailout:
2163         return (retval);
2164 }
2165
2166 /*
2167  * Return a buffer to the reader thread when we have completed writing it.
2168  */
2169 int
2170 camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf)
2171 {
2172         struct kevent ke;
2173         int retval = 0;
2174
2175         /*
2176          * Setup the event to let the other thread know that we have
2177          * completed a buffer.
2178          */
2179         EV_SET(&ke, (uintptr_t)&dev->peer_dev->peer_done_queue, EVFILT_USER, 0,
2180                NOTE_TRIGGER, 0, NULL);
2181
2182         /*
2183          * Drop our lock and acquire the other thread's lock before
2184          * manipulating
2185          */
2186         pthread_mutex_unlock(&dev->mutex);
2187         pthread_mutex_lock(&dev->peer_dev->mutex);
2188
2189         /*
2190          * Put the buffer on the reader thread's peer done queue now that
2191          * we have completed it.
2192          */
2193         STAILQ_INSERT_TAIL(&dev->peer_dev->peer_done_queue, peer_buf,
2194                            work_links);
2195         dev->peer_dev->num_peer_done_queue++;
2196
2197         /*
2198          * Send an event to the peer thread to let it know that we've added
2199          * something to its peer done queue.
2200          */
2201         retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL);
2202         if (retval == -1)
2203                 warn("%s: unable to add peer_done_queue kevent", __func__);
2204         else
2205                 retval = 0;
2206
2207         /*
2208          * Drop the other thread's lock and reacquire ours.
2209          */
2210         pthread_mutex_unlock(&dev->peer_dev->mutex);
2211         pthread_mutex_lock(&dev->mutex);
2212
2213         return (retval);
2214 }
2215
2216 /*
2217  * Free a buffer that was written out by the writer thread and returned to
2218  * the reader thread.
2219  */
2220 void
2221 camdd_peer_done(struct camdd_buf *buf)
2222 {
2223         struct camdd_dev *dev;
2224         struct camdd_buf_data *data;
2225
2226         dev = buf->dev;
2227         if (buf->buf_type != CAMDD_BUF_DATA) {
2228                 errx(1, "%s: should have a data buffer, not an "
2229                     "indirect buffer", __func__);
2230         }
2231
2232         data = &buf->buf_type_spec.data;
2233
2234         STAILQ_REMOVE(&dev->peer_work_queue, buf, camdd_buf, links);
2235         dev->num_peer_work_queue--;
2236         dev->peer_bytes_queued -= (data->fill_len - data->resid);
2237
2238         if (buf->status == CAMDD_STATUS_EOF)
2239                 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF;
2240
2241         STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
2242 }
2243
2244 /*
2245  * Assumes caller holds the lock for this device.
2246  */
2247 void
2248 camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf,
2249                    int *error_count)
2250 {
2251         int retval = 0;
2252
2253         /*
2254          * If we're the reader, we need to send the completed I/O
2255          * to the writer.  If we're the writer, we need to just
2256          * free up resources, or let the reader know if we've
2257          * encountered an error.
2258          */
2259         if (dev->write_dev == 0) {
2260                 retval = camdd_queue_peer_buf(dev, buf);
2261                 if (retval != 0)
2262                         (*error_count)++;
2263         } else {
2264                 struct camdd_buf *tmp_buf, *next_buf;
2265
2266                 STAILQ_FOREACH_SAFE(tmp_buf, &buf->src_list, src_links,
2267                                     next_buf) {
2268                         struct camdd_buf *src_buf;
2269                         struct camdd_buf_indirect *indirect;
2270
2271                         STAILQ_REMOVE(&buf->src_list, tmp_buf,
2272                                       camdd_buf, src_links);
2273
2274                         tmp_buf->status = buf->status;
2275
2276                         if (tmp_buf->buf_type == CAMDD_BUF_DATA) {
2277                                 camdd_complete_peer_buf(dev, tmp_buf);
2278                                 continue;
2279                         }
2280
2281                         indirect = &tmp_buf->buf_type_spec.indirect;
2282                         src_buf = indirect->src_buf;
2283                         src_buf->refcount--;
2284                         /*
2285                          * XXX KDM we probably need to account for
2286                          * exactly how many bytes we were able to
2287                          * write.  Allocate the residual to the
2288                          * first N buffers?  Or just track the
2289                          * number of bytes written?  Right now the reader
2290                          * doesn't do anything with a residual.
2291                          */
2292                         src_buf->status = buf->status;
2293                         if (src_buf->refcount <= 0)
2294                                 camdd_complete_peer_buf(dev, src_buf);
2295                         STAILQ_INSERT_TAIL(&dev->free_indirect_queue,
2296                                            tmp_buf, links);
2297                 }
2298
2299                 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
2300         }
2301 }
2302
2303 /*
2304  * Fetch all completed commands from the pass(4) device.
2305  *
2306  * Returns the number of commands received, or -1 if any of the commands
2307  * completed with an error.  Returns 0 if no commands are available.
2308  */
2309 int
2310 camdd_pass_fetch(struct camdd_dev *dev)
2311 {
2312         struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass;
2313         union ccb ccb;
2314         int retval = 0, num_fetched = 0, error_count = 0;
2315
2316         pthread_mutex_unlock(&dev->mutex);
2317         /*
2318          * XXX KDM we don't distinguish between EFAULT and ENOENT.
2319          */
2320         while ((retval = ioctl(pass_dev->dev->fd, CAMIOGET, &ccb)) != -1) {
2321                 struct camdd_buf *buf;
2322                 struct camdd_buf_data *data;
2323                 cam_status ccb_status;
2324                 union ccb *buf_ccb;
2325
2326                 buf = ccb.ccb_h.ccb_buf;
2327                 data = &buf->buf_type_spec.data;
2328                 buf_ccb = &data->ccb;
2329
2330                 num_fetched++;
2331
2332                 /*
2333                  * Copy the CCB back out so we get status, sense data, etc.
2334                  */
2335                 bcopy(&ccb, buf_ccb, sizeof(ccb));
2336
2337                 pthread_mutex_lock(&dev->mutex);
2338
2339                 /*
2340                  * We're now done, so take this off the active queue.
2341                  */
2342                 STAILQ_REMOVE(&dev->active_queue, buf, camdd_buf, links);
2343                 dev->cur_active_io--;
2344
2345                 ccb_status = ccb.ccb_h.status & CAM_STATUS_MASK;
2346                 if (ccb_status != CAM_REQ_CMP) {
2347                         cam_error_print(pass_dev->dev, &ccb, CAM_ESF_ALL,
2348                                         CAM_EPF_ALL, stderr);
2349                 }
2350
2351                 switch (pass_dev->protocol) {
2352                 case PROTO_SCSI:
2353                         data->resid = ccb.csio.resid;
2354                         dev->bytes_transferred += (ccb.csio.dxfer_len - ccb.csio.resid);
2355                         break;
2356                 case PROTO_NVME:
2357                         data->resid = 0;
2358                         dev->bytes_transferred += ccb.nvmeio.dxfer_len;
2359                         break;
2360                 default:
2361                         return -1;
2362                         break;
2363                 }
2364
2365                 if (buf->status == CAMDD_STATUS_NONE)
2366                         buf->status = camdd_ccb_status(&ccb, pass_dev->protocol);
2367                 if (buf->status == CAMDD_STATUS_ERROR)
2368                         error_count++;
2369                 else if (buf->status == CAMDD_STATUS_EOF) {
2370                         /*
2371                          * Once we queue this buffer to our partner thread,
2372                          * he will know that we've hit EOF.
2373                          */
2374                         dev->flags |= CAMDD_DEV_FLAG_EOF;
2375                 }
2376
2377                 camdd_complete_buf(dev, buf, &error_count);
2378
2379                 /*
2380                  * Unlock in preparation for the ioctl call.
2381                  */
2382                 pthread_mutex_unlock(&dev->mutex);
2383         }
2384
2385         pthread_mutex_lock(&dev->mutex);
2386
2387         if (error_count > 0)
2388                 return (-1);
2389         else
2390                 return (num_fetched);
2391 }
2392
2393 /*
2394  * Returns -1 for error, 0 for success/continue, and 1 for resource
2395  * shortage/stop processing.
2396  */
2397 int
2398 camdd_file_run(struct camdd_dev *dev)
2399 {
2400         struct camdd_dev_file *file_dev = &dev->dev_spec.file;
2401         struct camdd_buf_data *data;
2402         struct camdd_buf *buf;
2403         off_t io_offset;
2404         int retval = 0, write_dev = dev->write_dev;
2405         int error_count = 0, no_resources = 0, double_buf_needed = 0;
2406         uint32_t num_sectors = 0, db_len = 0;
2407
2408         buf = STAILQ_FIRST(&dev->run_queue);
2409         if (buf == NULL) {
2410                 no_resources = 1;
2411                 goto bailout;
2412         } else if ((dev->write_dev == 0)
2413                 && (dev->flags & (CAMDD_DEV_FLAG_EOF |
2414                                   CAMDD_DEV_FLAG_EOF_SENT))) {
2415                 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links);
2416                 dev->num_run_queue--;
2417                 buf->status = CAMDD_STATUS_EOF;
2418                 error_count++;
2419                 goto bailout;
2420         }
2421
2422         /*
2423          * If we're writing, we need to go through the source buffer list
2424          * and create an S/G list.
2425          */
2426         if (write_dev != 0) {
2427                 retval = camdd_buf_sg_create(buf, /*iovec*/ 1,
2428                     dev->sector_size, &num_sectors, &double_buf_needed);
2429                 if (retval != 0) {
2430                         no_resources = 1;
2431                         goto bailout;
2432                 }
2433         }
2434
2435         STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links);
2436         dev->num_run_queue--;
2437
2438         data = &buf->buf_type_spec.data;
2439
2440         /*
2441          * pread(2) and pwrite(2) offsets are byte offsets.
2442          */
2443         io_offset = buf->lba * dev->sector_size;
2444
2445         /*
2446          * Unlock the mutex while we read or write.
2447          */
2448         pthread_mutex_unlock(&dev->mutex);
2449
2450         /*
2451          * Note that we don't need to double buffer if we're the reader
2452          * because in that case, we have allocated a single buffer of
2453          * sufficient size to do the read.  This copy is necessary on
2454          * writes because if one of the components of the S/G list is not
2455          * a sector size multiple, the kernel will reject the write.  This
2456          * is unfortunate but not surprising.  So this will make sure that
2457          * we're using a single buffer that is a multiple of the sector size.
2458          */
2459         if ((double_buf_needed != 0)
2460          && (data->sg_count > 1)
2461          && (write_dev != 0)) {
2462                 uint32_t cur_offset;
2463                 int i;
2464
2465                 if (file_dev->tmp_buf == NULL)
2466                         file_dev->tmp_buf = calloc(dev->blocksize, 1);
2467                 if (file_dev->tmp_buf == NULL) {
2468                         buf->status = CAMDD_STATUS_ERROR;
2469                         error_count++;
2470                         pthread_mutex_lock(&dev->mutex);
2471                         goto bailout;
2472                 }
2473                 for (i = 0, cur_offset = 0; i < data->sg_count; i++) {
2474                         bcopy(data->iovec[i].iov_base,
2475                             &file_dev->tmp_buf[cur_offset],
2476                             data->iovec[i].iov_len);
2477                         cur_offset += data->iovec[i].iov_len;
2478                 }
2479                 db_len = cur_offset;
2480         }
2481
2482         if (file_dev->file_flags & CAMDD_FF_CAN_SEEK) {
2483                 if (write_dev == 0) {
2484                         /*
2485                          * XXX KDM is there any way we would need a S/G
2486                          * list here?
2487                          */
2488                         retval = pread(file_dev->fd, data->buf,
2489                             buf->len, io_offset);
2490                 } else {
2491                         if (double_buf_needed != 0) {
2492                                 retval = pwrite(file_dev->fd, file_dev->tmp_buf,
2493                                     db_len, io_offset);
2494                         } else if (data->sg_count == 0) {
2495                                 retval = pwrite(file_dev->fd, data->buf,
2496                                     data->fill_len, io_offset);
2497                         } else {
2498                                 retval = pwritev(file_dev->fd, data->iovec,
2499                                     data->sg_count, io_offset);
2500                         }
2501                 }
2502         } else {
2503                 if (write_dev == 0) {
2504                         /*
2505                          * XXX KDM is there any way we would need a S/G
2506                          * list here?
2507                          */
2508                         retval = read(file_dev->fd, data->buf, buf->len);
2509                 } else {
2510                         if (double_buf_needed != 0) {
2511                                 retval = write(file_dev->fd, file_dev->tmp_buf,
2512                                     db_len);
2513                         } else if (data->sg_count == 0) {
2514                                 retval = write(file_dev->fd, data->buf,
2515                                     data->fill_len);
2516                         } else {
2517                                 retval = writev(file_dev->fd, data->iovec,
2518                                     data->sg_count);
2519                         }
2520                 }
2521         }
2522
2523         /* We're done, re-acquire the lock */
2524         pthread_mutex_lock(&dev->mutex);
2525
2526         if (retval >= (ssize_t)data->fill_len) {
2527                 /*
2528                  * If the bytes transferred is more than the request size,
2529                  * that indicates an overrun, which should only happen at
2530                  * the end of a transfer if we have to round up to a sector
2531                  * boundary.
2532                  */
2533                 if (buf->status == CAMDD_STATUS_NONE)
2534                         buf->status = CAMDD_STATUS_OK;
2535                 data->resid = 0;
2536                 dev->bytes_transferred += retval;
2537         } else if (retval == -1) {
2538                 warn("Error %s %s", (write_dev) ? "writing to" :
2539                     "reading from", file_dev->filename);
2540
2541                 buf->status = CAMDD_STATUS_ERROR;
2542                 data->resid = data->fill_len;
2543                 error_count++;
2544
2545                 if (dev->debug == 0)
2546                         goto bailout;
2547
2548                 if ((double_buf_needed != 0)
2549                  && (write_dev != 0)) {
2550                         fprintf(stderr, "%s: fd %d, DB buf %p, len %u lba %ju "
2551                             "offset %ju\n", __func__, file_dev->fd,
2552                             file_dev->tmp_buf, db_len, (uintmax_t)buf->lba,
2553                             (uintmax_t)io_offset);
2554                 } else if (data->sg_count == 0) {
2555                         fprintf(stderr, "%s: fd %d, buf %p, len %u, lba %ju "
2556                             "offset %ju\n", __func__, file_dev->fd, data->buf,
2557                             data->fill_len, (uintmax_t)buf->lba,
2558                             (uintmax_t)io_offset);
2559                 } else {
2560                         int i;
2561
2562                         fprintf(stderr, "%s: fd %d, len %u, lba %ju "
2563                             "offset %ju\n", __func__, file_dev->fd,
2564                             data->fill_len, (uintmax_t)buf->lba,
2565                             (uintmax_t)io_offset);
2566
2567                         for (i = 0; i < data->sg_count; i++) {
2568                                 fprintf(stderr, "index %d ptr %p len %zu\n",
2569                                     i, data->iovec[i].iov_base,
2570                                     data->iovec[i].iov_len);
2571                         }
2572                 }
2573         } else if (retval == 0) {
2574                 buf->status = CAMDD_STATUS_EOF;
2575                 if (dev->debug != 0)
2576                         printf("%s: got EOF from %s!\n", __func__,
2577                             file_dev->filename);
2578                 data->resid = data->fill_len;
2579                 error_count++;
2580         } else if (retval < (ssize_t)data->fill_len) {
2581                 if (buf->status == CAMDD_STATUS_NONE)
2582                         buf->status = CAMDD_STATUS_SHORT_IO;
2583                 data->resid = data->fill_len - retval;
2584                 dev->bytes_transferred += retval;
2585         }
2586
2587 bailout:
2588         if (buf != NULL) {
2589                 if (buf->status == CAMDD_STATUS_EOF) {
2590                         struct camdd_buf *buf2;
2591                         dev->flags |= CAMDD_DEV_FLAG_EOF;
2592                         STAILQ_FOREACH(buf2, &dev->run_queue, links)
2593                                 buf2->status = CAMDD_STATUS_EOF;
2594                 }
2595
2596                 camdd_complete_buf(dev, buf, &error_count);
2597         }
2598
2599         if (error_count != 0)
2600                 return (-1);
2601         else if (no_resources != 0)
2602                 return (1);
2603         else
2604                 return (0);
2605 }
2606
2607 /*
2608  * Execute one command from the run queue.  Returns 0 for success, 1 for
2609  * stop processing, and -1 for error.
2610  */
2611 int
2612 camdd_pass_run(struct camdd_dev *dev)
2613 {
2614         struct camdd_buf *buf = NULL;
2615         struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass;
2616         struct camdd_buf_data *data;
2617         uint32_t num_blocks, sectors_used = 0;
2618         union ccb *ccb;
2619         int retval = 0, is_write = dev->write_dev;
2620         int double_buf_needed = 0;
2621
2622         buf = STAILQ_FIRST(&dev->run_queue);
2623         if (buf == NULL) {
2624                 retval = 1;
2625                 goto bailout;
2626         }
2627
2628         /*
2629          * If we're writing, we need to go through the source buffer list
2630          * and create an S/G list.
2631          */
2632         if (is_write != 0) {
2633                 retval = camdd_buf_sg_create(buf, /*iovec*/ 0,dev->sector_size,
2634                     &sectors_used, &double_buf_needed);
2635                 if (retval != 0) {
2636                         retval = -1;
2637                         goto bailout;
2638                 }
2639         }
2640
2641         STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links);
2642         dev->num_run_queue--;
2643
2644         data = &buf->buf_type_spec.data;
2645
2646         /*
2647          * In almost every case the number of blocks should be the device
2648          * block size.  The exception may be at the end of an I/O stream
2649          * for a partial block or at the end of a device.
2650          */
2651         if (is_write != 0)
2652                 num_blocks = sectors_used;
2653         else
2654                 num_blocks = data->fill_len / pass_dev->block_len;
2655
2656         ccb = &data->ccb;
2657
2658         switch (pass_dev->protocol) {
2659         case PROTO_SCSI:
2660                 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->csio);
2661
2662                 scsi_read_write(&ccb->csio,
2663                                 /*retries*/ dev->retry_count,
2664                                 /*cbfcnp*/ NULL,
2665                                 /*tag_action*/ MSG_SIMPLE_Q_TAG,
2666                                 /*readop*/ (dev->write_dev == 0) ? SCSI_RW_READ :
2667                                            SCSI_RW_WRITE,
2668                                 /*byte2*/ 0,
2669                                 /*minimum_cmd_size*/ dev->min_cmd_size,
2670                                 /*lba*/ buf->lba,
2671                                 /*block_count*/ num_blocks,
2672                                 /*data_ptr*/ (data->sg_count != 0) ?
2673                                              (uint8_t *)data->segs : data->buf,
2674                                 /*dxfer_len*/ (num_blocks * pass_dev->block_len),
2675                                 /*sense_len*/ SSD_FULL_SIZE,
2676                                 /*timeout*/ dev->io_timeout);
2677
2678                 if (data->sg_count != 0) {
2679                         ccb->csio.sglist_cnt = data->sg_count;
2680                 }
2681                 break;
2682         case PROTO_NVME:
2683                 CCB_CLEAR_ALL_EXCEPT_HDR(&ccb->nvmeio);
2684
2685                 nvme_read_write(&ccb->nvmeio,
2686                                 /*retries*/ dev->retry_count,
2687                                 /*cbfcnp*/ NULL,
2688                                 /*nsid*/ pass_dev->dev->target_lun & UINT32_MAX,
2689                                 /*readop*/ dev->write_dev == 0,
2690                                 /*lba*/ buf->lba,
2691                                 /*block_count*/ num_blocks,
2692                                 /*data_ptr*/ (data->sg_count != 0) ?
2693                                              (uint8_t *)data->segs : data->buf,
2694                                 /*dxfer_len*/ (num_blocks * pass_dev->block_len),
2695                                 /*timeout*/ dev->io_timeout);
2696
2697                 ccb->nvmeio.sglist_cnt = data->sg_count;
2698                 break;
2699         default:
2700                 retval = -1;
2701                 goto bailout;
2702         }
2703
2704         /* Disable freezing the device queue */
2705         ccb->ccb_h.flags |= CAM_DEV_QFRZDIS;
2706
2707         if (dev->retry_count != 0)
2708                 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
2709
2710         if (data->sg_count != 0) {
2711                 ccb->ccb_h.flags |= CAM_DATA_SG;
2712         }
2713
2714         /*
2715          * Store a pointer to the buffer in the CCB.  The kernel will
2716          * restore this when we get it back, and we'll use it to identify
2717          * the buffer this CCB came from.
2718          */
2719         ccb->ccb_h.ccb_buf = buf;
2720
2721         /*
2722          * Unlock our mutex in preparation for issuing the ioctl.
2723          */
2724         pthread_mutex_unlock(&dev->mutex);
2725         /*
2726          * Queue the CCB to the pass(4) driver.
2727          */
2728         if (ioctl(pass_dev->dev->fd, CAMIOQUEUE, ccb) == -1) {
2729                 pthread_mutex_lock(&dev->mutex);
2730
2731                 warn("%s: error sending CAMIOQUEUE ioctl to %s%u", __func__,
2732                      pass_dev->dev->device_name, pass_dev->dev->dev_unit_num);
2733                 warn("%s: CCB address is %p", __func__, ccb);
2734                 retval = -1;
2735
2736                 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
2737         } else {
2738                 pthread_mutex_lock(&dev->mutex);
2739
2740                 dev->cur_active_io++;
2741                 STAILQ_INSERT_TAIL(&dev->active_queue, buf, links);
2742         }
2743
2744 bailout:
2745         return (retval);
2746 }
2747
2748 int
2749 camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len)
2750 {
2751         struct camdd_dev_pass *pass_dev;
2752         uint32_t num_blocks;
2753         int retval = 0;
2754
2755         pass_dev = &dev->dev_spec.pass;
2756
2757         *lba = dev->next_io_pos_bytes / dev->sector_size;
2758         *len = dev->blocksize;
2759         num_blocks = *len / dev->sector_size;
2760
2761         /*
2762          * If max_sector is 0, then we have no set limit.  This can happen
2763          * if we're writing to a file in a filesystem, or reading from
2764          * something like /dev/zero.
2765          */
2766         if ((dev->max_sector != 0)
2767          || (dev->sector_io_limit != 0)) {
2768                 uint64_t max_sector;
2769
2770                 if ((dev->max_sector != 0)
2771                  && (dev->sector_io_limit != 0))
2772                         max_sector = min(dev->sector_io_limit, dev->max_sector);
2773                 else if (dev->max_sector != 0)
2774                         max_sector = dev->max_sector;
2775                 else
2776                         max_sector = dev->sector_io_limit;
2777
2778
2779                 /*
2780                  * Check to see whether we're starting off past the end of
2781                  * the device.  If so, we need to just send an EOF
2782                  * notification to the writer.
2783                  */
2784                 if (*lba > max_sector) {
2785                         *len = 0;
2786                         retval = 1;
2787                 } else if (((*lba + num_blocks) > max_sector + 1)
2788                         || ((*lba + num_blocks) < *lba)) {
2789                         /*
2790                          * If we get here (but pass the first check), we
2791                          * can trim the request length down to go to the
2792                          * end of the device.
2793                          */
2794                         num_blocks = (max_sector + 1) - *lba;
2795                         *len = num_blocks * dev->sector_size;
2796                         retval = 1;
2797                 }
2798         }
2799
2800         dev->next_io_pos_bytes += *len;
2801
2802         return (retval);
2803 }
2804
2805 /*
2806  * Returns 0 for success, 1 for EOF detected, and -1 for failure.
2807  */
2808 int
2809 camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf)
2810 {
2811         struct camdd_buf *buf = NULL;
2812         struct camdd_buf_data *data;
2813         struct camdd_dev_pass *pass_dev;
2814         size_t new_len;
2815         struct camdd_buf_data *rb_data;
2816         int is_write = dev->write_dev;
2817         int eof_flush_needed = 0;
2818         int retval = 0;
2819         int error;
2820
2821         pass_dev = &dev->dev_spec.pass;
2822
2823         /*
2824          * If we've gotten EOF or our partner has, we should not continue
2825          * queueing I/O.  If we're a writer, though, we should continue
2826          * to write any buffers that don't have EOF status.
2827          */
2828         if ((dev->flags & CAMDD_DEV_FLAG_EOF)
2829          || ((dev->flags & CAMDD_DEV_FLAG_PEER_EOF)
2830           && (is_write == 0))) {
2831                 /*
2832                  * Tell the worker thread that we have seen EOF.
2833                  */
2834                 retval = 1;
2835
2836                 /*
2837                  * If we're the writer, send the buffer back with EOF status.
2838                  */
2839                 if (is_write) {
2840                         read_buf->status = CAMDD_STATUS_EOF;
2841
2842                         error = camdd_complete_peer_buf(dev, read_buf);
2843                 }
2844                 goto bailout;
2845         }
2846
2847         if (is_write == 0) {
2848                 buf = camdd_get_buf(dev, CAMDD_BUF_DATA);
2849                 if (buf == NULL) {
2850                         retval = -1;
2851                         goto bailout;
2852                 }
2853                 data = &buf->buf_type_spec.data;
2854
2855                 retval = camdd_get_next_lba_len(dev, &buf->lba, &buf->len);
2856                 if (retval != 0) {
2857                         buf->status = CAMDD_STATUS_EOF;
2858
2859                         if ((buf->len == 0)
2860                          && ((dev->flags & (CAMDD_DEV_FLAG_EOF_SENT |
2861                              CAMDD_DEV_FLAG_EOF_QUEUED)) != 0)) {
2862                                 camdd_release_buf(buf);
2863                                 goto bailout;
2864                         }
2865                         dev->flags |= CAMDD_DEV_FLAG_EOF_QUEUED;
2866                 }
2867
2868                 data->fill_len = buf->len;
2869                 data->src_start_offset = buf->lba * dev->sector_size;
2870
2871                 /*
2872                  * Put this on the run queue.
2873                  */
2874                 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
2875                 dev->num_run_queue++;
2876
2877                 /* We're done. */
2878                 goto bailout;
2879         }
2880
2881         /*
2882          * Check for new EOF status from the reader.
2883          */
2884         if ((read_buf->status == CAMDD_STATUS_EOF)
2885          || (read_buf->status == CAMDD_STATUS_ERROR)) {
2886                 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF;
2887                 if ((STAILQ_FIRST(&dev->pending_queue) == NULL)
2888                  && (read_buf->len == 0)) {
2889                         camdd_complete_peer_buf(dev, read_buf);
2890                         retval = 1;
2891                         goto bailout;
2892                 } else
2893                         eof_flush_needed = 1;
2894         }
2895
2896         /*
2897          * See if we have a buffer we're composing with pieces from our
2898          * partner thread.
2899          */
2900         buf = STAILQ_FIRST(&dev->pending_queue);
2901         if (buf == NULL) {
2902                 uint64_t lba;
2903                 ssize_t len;
2904
2905                 retval = camdd_get_next_lba_len(dev, &lba, &len);
2906                 if (retval != 0) {
2907                         read_buf->status = CAMDD_STATUS_EOF;
2908
2909                         if (len == 0) {
2910                                 dev->flags |= CAMDD_DEV_FLAG_EOF;
2911                                 error = camdd_complete_peer_buf(dev, read_buf);
2912                                 goto bailout;
2913                         }
2914                 }
2915
2916                 /*
2917                  * If we don't have a pending buffer, we need to grab a new
2918                  * one from the free list or allocate another one.
2919                  */
2920                 buf = camdd_get_buf(dev, CAMDD_BUF_DATA);
2921                 if (buf == NULL) {
2922                         retval = 1;
2923                         goto bailout;
2924                 }
2925
2926                 buf->lba = lba;
2927                 buf->len = len;
2928
2929                 STAILQ_INSERT_TAIL(&dev->pending_queue, buf, links);
2930                 dev->num_pending_queue++;
2931         }
2932
2933         data = &buf->buf_type_spec.data;
2934
2935         rb_data = &read_buf->buf_type_spec.data;
2936
2937         if ((rb_data->src_start_offset != dev->next_peer_pos_bytes)
2938          && (dev->debug != 0)) {
2939                 printf("%s: WARNING: reader offset %#jx != expected offset "
2940                     "%#jx\n", __func__, (uintmax_t)rb_data->src_start_offset,
2941                     (uintmax_t)dev->next_peer_pos_bytes);
2942         }
2943         dev->next_peer_pos_bytes = rb_data->src_start_offset +
2944             (rb_data->fill_len - rb_data->resid);
2945
2946         new_len = (rb_data->fill_len - rb_data->resid) + data->fill_len;
2947         if (new_len < buf->len) {
2948                 /*
2949                  * There are three cases here:
2950                  * 1. We need more data to fill up a block, so we put
2951                  *    this I/O on the queue and wait for more I/O.
2952                  * 2. We have a pending buffer in the queue that is
2953                  *    smaller than our blocksize, but we got an EOF.  So we
2954                  *    need to go ahead and flush the write out.
2955                  * 3. We got an error.
2956                  */
2957
2958                 /*
2959                  * Increment our fill length.
2960                  */
2961                 data->fill_len += (rb_data->fill_len - rb_data->resid);
2962
2963                 /*
2964                  * Add the new read buffer to the list for writing.
2965                  */
2966                 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links);
2967
2968                 /* Increment the count */
2969                 buf->src_count++;
2970
2971                 if (eof_flush_needed == 0) {
2972                         /*
2973                          * We need to exit, because we don't have enough
2974                          * data yet.
2975                          */
2976                         goto bailout;
2977                 } else {
2978                         /*
2979                          * Take the buffer off of the pending queue.
2980                          */
2981                         STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf,
2982                                       links);
2983                         dev->num_pending_queue--;
2984
2985                         /*
2986                          * If we need an EOF flush, but there is no data
2987                          * to flush, go ahead and return this buffer.
2988                          */
2989                         if (data->fill_len == 0) {
2990                                 camdd_complete_buf(dev, buf, /*error_count*/0);
2991                                 retval = 1;
2992                                 goto bailout;
2993                         }
2994
2995                         /*
2996                          * Put this on the next queue for execution.
2997                          */
2998                         STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
2999                         dev->num_run_queue++;
3000                 }
3001         } else if (new_len == buf->len) {
3002                 /*
3003                  * We have enough data to completey fill one block,
3004                  * so we're ready to issue the I/O.
3005                  */
3006
3007                 /*
3008                  * Take the buffer off of the pending queue.
3009                  */
3010                 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, links);
3011                 dev->num_pending_queue--;
3012
3013                 /*
3014                  * Add the new read buffer to the list for writing.
3015                  */
3016                 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links);
3017
3018                 /* Increment the count */
3019                 buf->src_count++;
3020
3021                 /*
3022                  * Increment our fill length.
3023                  */
3024                 data->fill_len += (rb_data->fill_len - rb_data->resid);
3025
3026                 /*
3027                  * Put this on the next queue for execution.
3028                  */
3029                 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
3030                 dev->num_run_queue++;
3031         } else {
3032                 struct camdd_buf *idb;
3033                 struct camdd_buf_indirect *indirect;
3034                 uint32_t len_to_go, cur_offset;
3035
3036
3037                 idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT);
3038                 if (idb == NULL) {
3039                         retval = 1;
3040                         goto bailout;
3041                 }
3042                 indirect = &idb->buf_type_spec.indirect;
3043                 indirect->src_buf = read_buf;
3044                 read_buf->refcount++;
3045                 indirect->offset = 0;
3046                 indirect->start_ptr = rb_data->buf;
3047                 /*
3048                  * We've already established that there is more
3049                  * data in read_buf than we have room for in our
3050                  * current write request.  So this particular chunk
3051                  * of the request should just be the remainder
3052                  * needed to fill up a block.
3053                  */
3054                 indirect->len = buf->len - (data->fill_len - data->resid);
3055
3056                 camdd_buf_add_child(buf, idb);
3057
3058                 /*
3059                  * This buffer is ready to execute, so we can take
3060                  * it off the pending queue and put it on the run
3061                  * queue.
3062                  */
3063                 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf,
3064                               links);
3065                 dev->num_pending_queue--;
3066                 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
3067                 dev->num_run_queue++;
3068
3069                 cur_offset = indirect->offset + indirect->len;
3070
3071                 /*
3072                  * The resulting I/O would be too large to fit in
3073                  * one block.  We need to split this I/O into
3074                  * multiple pieces.  Allocate as many buffers as needed.
3075                  */
3076                 for (len_to_go = rb_data->fill_len - rb_data->resid -
3077                      indirect->len; len_to_go > 0;) {
3078                         struct camdd_buf *new_buf;
3079                         struct camdd_buf_data *new_data;
3080                         uint64_t lba;
3081                         ssize_t len;
3082
3083                         retval = camdd_get_next_lba_len(dev, &lba, &len);
3084                         if ((retval != 0)
3085                          && (len == 0)) {
3086                                 /*
3087                                  * The device has already been marked
3088                                  * as EOF, and there is no space left.
3089                                  */
3090                                 goto bailout;
3091                         }
3092
3093                         new_buf = camdd_get_buf(dev, CAMDD_BUF_DATA);
3094                         if (new_buf == NULL) {
3095                                 retval = 1;
3096                                 goto bailout;
3097                         }
3098
3099                         new_buf->lba = lba;
3100                         new_buf->len = len;
3101
3102                         idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT);
3103                         if (idb == NULL) {
3104                                 retval = 1;
3105                                 goto bailout;
3106                         }
3107
3108                         indirect = &idb->buf_type_spec.indirect;
3109
3110                         indirect->src_buf = read_buf;
3111                         read_buf->refcount++;
3112                         indirect->offset = cur_offset;
3113                         indirect->start_ptr = rb_data->buf + cur_offset;
3114                         indirect->len = min(len_to_go, new_buf->len);
3115 #if 0
3116                         if (((indirect->len % dev->sector_size) != 0)
3117                          || ((indirect->offset % dev->sector_size) != 0)) {
3118                                 warnx("offset %ju len %ju not aligned with "
3119                                     "sector size %u", indirect->offset,
3120                                     (uintmax_t)indirect->len, dev->sector_size);
3121                         }
3122 #endif
3123                         cur_offset += indirect->len;
3124                         len_to_go -= indirect->len;
3125
3126                         camdd_buf_add_child(new_buf, idb);
3127
3128                         new_data = &new_buf->buf_type_spec.data;
3129
3130                         if ((new_data->fill_len == new_buf->len)
3131                          || (eof_flush_needed != 0)) {
3132                                 STAILQ_INSERT_TAIL(&dev->run_queue,
3133                                                    new_buf, links);
3134                                 dev->num_run_queue++;
3135                         } else if (new_data->fill_len < buf->len) {
3136                                 STAILQ_INSERT_TAIL(&dev->pending_queue,
3137                                                 new_buf, links);
3138                                 dev->num_pending_queue++;
3139                         } else {
3140                                 warnx("%s: too much data in new "
3141                                       "buffer!", __func__);
3142                                 retval = 1;
3143                                 goto bailout;
3144                         }
3145                 }
3146         }
3147
3148 bailout:
3149         return (retval);
3150 }
3151
3152 void
3153 camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth,
3154                 uint32_t *peer_depth, uint32_t *our_bytes, uint32_t *peer_bytes)
3155 {
3156         *our_depth = dev->cur_active_io + dev->num_run_queue;
3157         if (dev->num_peer_work_queue >
3158             dev->num_peer_done_queue)
3159                 *peer_depth = dev->num_peer_work_queue -
3160                               dev->num_peer_done_queue;
3161         else
3162                 *peer_depth = 0;
3163         *our_bytes = *our_depth * dev->blocksize;
3164         *peer_bytes = dev->peer_bytes_queued;
3165 }
3166
3167 void
3168 camdd_sig_handler(int sig)
3169 {
3170         if (sig == SIGINFO)
3171                 need_status = 1;
3172         else {
3173                 need_exit = 1;
3174                 error_exit = 1;
3175         }
3176
3177         sem_post(&camdd_sem);
3178 }
3179
3180 void
3181 camdd_print_status(struct camdd_dev *camdd_dev, struct camdd_dev *other_dev,
3182                    struct timespec *start_time)
3183 {
3184         struct timespec done_time;
3185         uint64_t total_ns;
3186         long double mb_sec, total_sec;
3187         int error = 0;
3188
3189         error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &done_time);
3190         if (error != 0) {
3191                 warn("Unable to get done time");
3192                 return;
3193         }
3194
3195         timespecsub(&done_time, start_time, &done_time);
3196
3197         total_ns = done_time.tv_nsec + (done_time.tv_sec * 1000000000);
3198         total_sec = total_ns;
3199         total_sec /= 1000000000;
3200
3201         fprintf(stderr, "%ju bytes %s %s\n%ju bytes %s %s\n"
3202                 "%.4Lf seconds elapsed\n",
3203                 (uintmax_t)camdd_dev->bytes_transferred,
3204                 (camdd_dev->write_dev == 0) ?  "read from" : "written to",
3205                 camdd_dev->device_name,
3206                 (uintmax_t)other_dev->bytes_transferred,
3207                 (other_dev->write_dev == 0) ? "read from" : "written to",
3208                 other_dev->device_name, total_sec);
3209
3210         mb_sec = min(other_dev->bytes_transferred,camdd_dev->bytes_transferred);
3211         mb_sec /= 1024 * 1024;
3212         mb_sec *= 1000000000;
3213         mb_sec /= total_ns;
3214         fprintf(stderr, "%.2Lf MB/sec\n", mb_sec);
3215 }
3216
3217 int
3218 camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts, uint64_t max_io,
3219          int retry_count, int timeout)
3220 {
3221         struct cam_device *new_cam_dev = NULL;
3222         struct camdd_dev *devs[2];
3223         struct timespec start_time;
3224         pthread_t threads[2];
3225         int unit = 0;
3226         int error = 0;
3227         int i;
3228
3229         bzero(devs, sizeof(devs));
3230
3231         if (num_io_opts != 2) {
3232                 warnx("Must have one input and one output path");
3233                 error = 1;
3234                 goto bailout;
3235         }
3236
3237         for (i = 0; i < num_io_opts; i++) {
3238                 switch (io_opts[i].dev_type) {
3239                 case CAMDD_DEV_PASS: {
3240                         if (isdigit(io_opts[i].dev_name[0])) {
3241                                 camdd_argmask new_arglist = CAMDD_ARG_NONE;
3242                                 int bus = 0, target = 0, lun = 0;
3243                                 int rv;
3244
3245                                 /* device specified as bus:target[:lun] */
3246                                 rv = parse_btl(io_opts[i].dev_name, &bus,
3247                                     &target, &lun, &new_arglist);
3248                                 if (rv < 2) {
3249                                         warnx("numeric device specification "
3250                                              "must be either bus:target, or "
3251                                              "bus:target:lun");
3252                                         error = 1;
3253                                         goto bailout;
3254                                 }
3255                                 /* default to 0 if lun was not specified */
3256                                 if ((new_arglist & CAMDD_ARG_LUN) == 0) {
3257                                         lun = 0;
3258                                         new_arglist |= CAMDD_ARG_LUN;
3259                                 }
3260                                 new_cam_dev = cam_open_btl(bus, target, lun,
3261                                     O_RDWR, NULL);
3262                         } else {
3263                                 char name[30];
3264
3265                                 if (cam_get_device(io_opts[i].dev_name, name,
3266                                                    sizeof name, &unit) == -1) {
3267                                         warnx("%s", cam_errbuf);
3268                                         error = 1;
3269                                         goto bailout;
3270                                 }
3271                                 new_cam_dev = cam_open_spec_device(name, unit,
3272                                     O_RDWR, NULL);
3273                         }
3274
3275                         if (new_cam_dev == NULL) {
3276                                 warnx("%s", cam_errbuf);
3277                                 error = 1;
3278                                 goto bailout;
3279                         }
3280
3281                         devs[i] = camdd_probe_pass(new_cam_dev,
3282                             /*io_opts*/ &io_opts[i],
3283                             CAMDD_ARG_ERR_RECOVER,
3284                             /*probe_retry_count*/ 3,
3285                             /*probe_timeout*/ 5000,
3286                             /*io_retry_count*/ retry_count,
3287                             /*io_timeout*/ timeout);
3288                         if (devs[i] == NULL) {
3289                                 warn("Unable to probe device %s%u",
3290                                      new_cam_dev->device_name,
3291                                      new_cam_dev->dev_unit_num);
3292                                 error = 1;
3293                                 goto bailout;
3294                         }
3295                         break;
3296                 }
3297                 case CAMDD_DEV_FILE: {
3298                         int fd = -1;
3299
3300                         if (io_opts[i].dev_name[0] == '-') {
3301                                 if (io_opts[i].write_dev != 0)
3302                                         fd = STDOUT_FILENO;
3303                                 else
3304                                         fd = STDIN_FILENO;
3305                         } else {
3306                                 if (io_opts[i].write_dev != 0) {
3307                                         fd = open(io_opts[i].dev_name,
3308                                             O_RDWR | O_CREAT, S_IWUSR |S_IRUSR);
3309                                 } else {
3310                                         fd = open(io_opts[i].dev_name,
3311                                             O_RDONLY);
3312                                 }
3313                         }
3314                         if (fd == -1) {
3315                                 warn("error opening file %s",
3316                                     io_opts[i].dev_name);
3317                                 error = 1;
3318                                 goto bailout;
3319                         }
3320
3321                         devs[i] = camdd_probe_file(fd, &io_opts[i],
3322                             retry_count, timeout);
3323                         if (devs[i] == NULL) {
3324                                 error = 1;
3325                                 goto bailout;
3326                         }
3327
3328                         break;
3329                 }
3330                 default:
3331                         warnx("Unknown device type %d (%s)",
3332                             io_opts[i].dev_type, io_opts[i].dev_name);
3333                         error = 1;
3334                         goto bailout;
3335                         break; /*NOTREACHED */
3336                 }
3337
3338                 devs[i]->write_dev = io_opts[i].write_dev;
3339
3340                 devs[i]->start_offset_bytes = io_opts[i].offset;
3341
3342                 if (max_io != 0) {
3343                         devs[i]->sector_io_limit =
3344                             (devs[i]->start_offset_bytes /
3345                             devs[i]->sector_size) +
3346                             (max_io / devs[i]->sector_size) - 1;
3347                 }
3348
3349                 devs[i]->next_io_pos_bytes = devs[i]->start_offset_bytes;
3350                 devs[i]->next_completion_pos_bytes =devs[i]->start_offset_bytes;
3351         }
3352
3353         devs[0]->peer_dev = devs[1];
3354         devs[1]->peer_dev = devs[0];
3355         devs[0]->next_peer_pos_bytes = devs[0]->peer_dev->next_io_pos_bytes;
3356         devs[1]->next_peer_pos_bytes = devs[1]->peer_dev->next_io_pos_bytes;
3357
3358         sem_init(&camdd_sem, /*pshared*/ 0, 0);
3359
3360         signal(SIGINFO, camdd_sig_handler);
3361         signal(SIGINT, camdd_sig_handler);
3362
3363         error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &start_time);
3364         if (error != 0) {
3365                 warn("Unable to get start time");
3366                 goto bailout;
3367         }
3368
3369         for (i = 0; i < num_io_opts; i++) {
3370                 error = pthread_create(&threads[i], NULL, camdd_worker,
3371                                        (void *)devs[i]);
3372                 if (error != 0) {
3373                         warnc(error, "pthread_create() failed");
3374                         goto bailout;
3375                 }
3376         }
3377
3378         for (;;) {
3379                 if ((sem_wait(&camdd_sem) == -1)
3380                  || (need_exit != 0)) {
3381                         struct kevent ke;
3382
3383                         for (i = 0; i < num_io_opts; i++) {
3384                                 EV_SET(&ke, (uintptr_t)&devs[i]->work_queue,
3385                                     EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
3386
3387                                 devs[i]->flags |= CAMDD_DEV_FLAG_EOF;
3388
3389                                 error = kevent(devs[i]->kq, &ke, 1, NULL, 0,
3390                                                 NULL);
3391                                 if (error == -1)
3392                                         warn("%s: unable to wake up thread",
3393                                             __func__);
3394                                 error = 0;
3395                         }
3396                         break;
3397                 } else if (need_status != 0) {
3398                         camdd_print_status(devs[0], devs[1], &start_time);
3399                         need_status = 0;
3400                 }
3401         }
3402         for (i = 0; i < num_io_opts; i++) {
3403                 pthread_join(threads[i], NULL);
3404         }
3405
3406         camdd_print_status(devs[0], devs[1], &start_time);
3407
3408 bailout:
3409
3410         for (i = 0; i < num_io_opts; i++)
3411                 camdd_free_dev(devs[i]);
3412
3413         return (error + error_exit);
3414 }
3415
3416 void
3417 usage(void)
3418 {
3419         fprintf(stderr,
3420 "usage:  camdd <-i|-o pass=pass0,bs=1M,offset=1M,depth=4>\n"
3421 "              <-i|-o file=/tmp/file,bs=512K,offset=1M>\n"
3422 "              <-i|-o file=/dev/da0,bs=512K,offset=1M>\n"
3423 "              <-i|-o file=/dev/nsa0,bs=512K>\n"
3424 "              [-C retry_count][-E][-m max_io_amt][-t timeout_secs][-v][-h]\n"
3425 "Option description\n"
3426 "-i <arg=val>  Specify input device/file and parameters\n"
3427 "-o <arg=val>  Specify output device/file and parameters\n"
3428 "Input and Output parameters\n"
3429 "pass=name     Specify a pass(4) device like pass0 or /dev/pass0\n"
3430 "file=name     Specify a file or device, /tmp/foo, /dev/da0, /dev/null\n"
3431 "              or - for stdin/stdout\n"
3432 "bs=blocksize  Specify blocksize in bytes, or using K, M, G, etc. suffix\n"
3433 "offset=len    Specify starting offset in bytes or using K, M, G suffix\n"
3434 "              NOTE: offset cannot be specified on tapes, pipes, stdin/out\n"
3435 "depth=N       Specify a numeric queue depth.  This only applies to pass(4)\n"
3436 "mcs=N         Specify a minimum cmd size for pass(4) read/write commands\n"
3437 "Optional arguments\n"
3438 "-C retry_cnt  Specify a retry count for pass(4) devices\n"
3439 "-E            Enable CAM error recovery for pass(4) devices\n"
3440 "-m max_io     Specify the maximum amount to be transferred in bytes or\n"
3441 "              using K, G, M, etc. suffixes\n"
3442 "-t timeout    Specify the I/O timeout to use with pass(4) devices\n"
3443 "-v            Enable verbose error recovery\n"
3444 "-h            Print this message\n");
3445 }
3446
3447
3448 int
3449 camdd_parse_io_opts(char *args, int is_write, struct camdd_io_opts *io_opts)
3450 {
3451         char *tmpstr, *tmpstr2;
3452         char *orig_tmpstr = NULL;
3453         int retval = 0;
3454
3455         io_opts->write_dev = is_write;
3456
3457         tmpstr = strdup(args);
3458         if (tmpstr == NULL) {
3459                 warn("strdup failed");
3460                 retval = 1;
3461                 goto bailout;
3462         }
3463         orig_tmpstr = tmpstr;
3464         while ((tmpstr2 = strsep(&tmpstr, ",")) != NULL) {
3465                 char *name, *value;
3466
3467                 /*
3468                  * If the user creates an empty parameter by putting in two
3469                  * commas, skip over it and look for the next field.
3470                  */
3471                 if (*tmpstr2 == '\0')
3472                         continue;
3473
3474                 name = strsep(&tmpstr2, "=");
3475                 if (*name == '\0') {
3476                         warnx("Got empty I/O parameter name");
3477                         retval = 1;
3478                         goto bailout;
3479                 }
3480                 value = strsep(&tmpstr2, "=");
3481                 if ((value == NULL)
3482                  || (*value == '\0')) {
3483                         warnx("Empty I/O parameter value for %s", name);
3484                         retval = 1;
3485                         goto bailout;
3486                 }
3487                 if (strncasecmp(name, "file", 4) == 0) {
3488                         io_opts->dev_type = CAMDD_DEV_FILE;
3489                         io_opts->dev_name = strdup(value);
3490                         if (io_opts->dev_name == NULL) {
3491                                 warn("Error allocating memory");
3492                                 retval = 1;
3493                                 goto bailout;
3494                         }
3495                 } else if (strncasecmp(name, "pass", 4) == 0) {
3496                         io_opts->dev_type = CAMDD_DEV_PASS;
3497                         io_opts->dev_name = strdup(value);
3498                         if (io_opts->dev_name == NULL) {
3499                                 warn("Error allocating memory");
3500                                 retval = 1;
3501                                 goto bailout;
3502                         }
3503                 } else if ((strncasecmp(name, "bs", 2) == 0)
3504                         || (strncasecmp(name, "blocksize", 9) == 0)) {
3505                         retval = expand_number(value, &io_opts->blocksize);
3506                         if (retval == -1) {
3507                                 warn("expand_number(3) failed on %s=%s", name,
3508                                     value);
3509                                 retval = 1;
3510                                 goto bailout;
3511                         }
3512                 } else if (strncasecmp(name, "depth", 5) == 0) {
3513                         char *endptr;
3514
3515                         io_opts->queue_depth = strtoull(value, &endptr, 0);
3516                         if (*endptr != '\0') {
3517                                 warnx("invalid queue depth %s", value);
3518                                 retval = 1;
3519                                 goto bailout;
3520                         }
3521                 } else if (strncasecmp(name, "mcs", 3) == 0) {
3522                         char *endptr;
3523
3524                         io_opts->min_cmd_size = strtol(value, &endptr, 0);
3525                         if ((*endptr != '\0')
3526                          || ((io_opts->min_cmd_size > 16)
3527                           || (io_opts->min_cmd_size < 0))) {
3528                                 warnx("invalid minimum cmd size %s", value);
3529                                 retval = 1;
3530                                 goto bailout;
3531                         }
3532                 } else if (strncasecmp(name, "offset", 6) == 0) {
3533                         retval = expand_number(value, &io_opts->offset);
3534                         if (retval == -1) {
3535                                 warn("expand_number(3) failed on %s=%s", name,
3536                                     value);
3537                                 retval = 1;
3538                                 goto bailout;
3539                         }
3540                 } else if (strncasecmp(name, "debug", 5) == 0) {
3541                         char *endptr;
3542
3543                         io_opts->debug = strtoull(value, &endptr, 0);
3544                         if (*endptr != '\0') {
3545                                 warnx("invalid debug level %s", value);
3546                                 retval = 1;
3547                                 goto bailout;
3548                         }
3549                 } else {
3550                         warnx("Unrecognized parameter %s=%s", name, value);
3551                 }
3552         }
3553 bailout:
3554         free(orig_tmpstr);
3555
3556         return (retval);
3557 }
3558
3559 int
3560 main(int argc, char **argv)
3561 {
3562         int c;
3563         camdd_argmask arglist = CAMDD_ARG_NONE;
3564         int timeout = 0, retry_count = 1;
3565         int error = 0;
3566         uint64_t max_io = 0;
3567         struct camdd_io_opts *opt_list = NULL;
3568
3569         if (argc == 1) {
3570                 usage();
3571                 exit(1);
3572         }
3573
3574         opt_list = calloc(2, sizeof(struct camdd_io_opts));
3575         if (opt_list == NULL) {
3576                 warn("Unable to allocate option list");
3577                 error = 1;
3578                 goto bailout;
3579         }
3580
3581         while ((c = getopt(argc, argv, "C:Ehi:m:o:t:v")) != -1){
3582                 switch (c) {
3583                 case 'C':
3584                         retry_count = strtol(optarg, NULL, 0);
3585                         if (retry_count < 0)
3586                                 errx(1, "retry count %d is < 0",
3587                                      retry_count);
3588                         arglist |= CAMDD_ARG_RETRIES;
3589                         break;
3590                 case 'E':
3591                         arglist |= CAMDD_ARG_ERR_RECOVER;
3592                         break;
3593                 case 'i':
3594                 case 'o':
3595                         if (((c == 'i')
3596                           && (opt_list[0].dev_type != CAMDD_DEV_NONE))
3597                          || ((c == 'o')
3598                           && (opt_list[1].dev_type != CAMDD_DEV_NONE))) {
3599                                 errx(1, "Only one input and output path "
3600                                     "allowed");
3601                         }
3602                         error = camdd_parse_io_opts(optarg, (c == 'o') ? 1 : 0,
3603                             (c == 'o') ? &opt_list[1] : &opt_list[0]);
3604                         if (error != 0)
3605                                 goto bailout;
3606                         break;
3607                 case 'm':
3608                         error = expand_number(optarg, &max_io);
3609                         if (error == -1) {
3610                                 warn("invalid maximum I/O amount %s", optarg);
3611                                 error = 1;
3612                                 goto bailout;
3613                         }
3614                         break;
3615                 case 't':
3616                         timeout = strtol(optarg, NULL, 0);
3617                         if (timeout < 0)
3618                                 errx(1, "invalid timeout %d", timeout);
3619                         /* Convert the timeout from seconds to ms */
3620                         timeout *= 1000;
3621                         arglist |= CAMDD_ARG_TIMEOUT;
3622                         break;
3623                 case 'v':
3624                         arglist |= CAMDD_ARG_VERBOSE;
3625                         break;
3626                 case 'h':
3627                 default:
3628                         usage();
3629                         exit(1);
3630                         break; /*NOTREACHED*/
3631                 }
3632         }
3633
3634         if ((opt_list[0].dev_type == CAMDD_DEV_NONE)
3635          || (opt_list[1].dev_type == CAMDD_DEV_NONE))
3636                 errx(1, "Must specify both -i and -o");
3637
3638         /*
3639          * Set the timeout if the user hasn't specified one.
3640          */
3641         if (timeout == 0)
3642                 timeout = CAMDD_PASS_RW_TIMEOUT;
3643
3644         error = camdd_rw(opt_list, 2, max_io, retry_count, timeout);
3645
3646 bailout:
3647         free(opt_list);
3648
3649         exit(error);
3650 }