usr.sbin/camdd/camdd.c

   1 /*-
   2  * Copyright (c) 1997-2007 Kenneth D. Merry
   3  * Copyright (c) 2013, 2014, 2015 Spectra Logic Corporation
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions, and the following disclaimer,
  11  *    without modification.
  12  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  13  *    substantially similar to the "NO WARRANTY" disclaimer below
  14  *    ("Disclaimer") and any redistribution must be conditioned upon
  15  *    including a substantially similar Disclaimer requirement for further
  16  *    binary redistribution.
  17  *
  18  * NO WARRANTY
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  22  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  27  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  28  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29  * POSSIBILITY OF SUCH DAMAGES.
  30  *
  31  * Authors: Ken Merry           (Spectra Logic Corporation)
  32  */
  33
  34 /*
  35  * This is eventually intended to be:
  36  * - A basic data transfer/copy utility
  37  * - A simple benchmark utility
  38  * - An example of how to use the asynchronous pass(4) driver interface.
  39  */
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include <sys/ioctl.h>
  44 #include <sys/stdint.h>
  45 #include <sys/types.h>
  46 #include <sys/endian.h>
  47 #include <sys/param.h>
  48 #include <sys/sbuf.h>
  49 #include <sys/stat.h>
  50 #include <sys/event.h>
  51 #include <sys/time.h>
  52 #include <sys/uio.h>
  53 #include <vm/vm.h>
  54 #include <machine/bus.h>
  55 #include <sys/bus.h>
  56 #include <sys/bus_dma.h>
  57 #include <sys/mtio.h>
  58 #include <sys/conf.h>
  59 #include <sys/disk.h>
  60
  61 #include <stdio.h>
  62 #include <stdlib.h>
  63 #include <semaphore.h>
  64 #include <string.h>
  65 #include <unistd.h>
  66 #include <inttypes.h>
  67 #include <limits.h>
  68 #include <fcntl.h>
  69 #include <ctype.h>
  70 #include <err.h>
  71 #include <libutil.h>
  72 #include <pthread.h>
  73 #include <assert.h>
  74 #include <bsdxml.h>
  75
  76 #include <cam/cam.h>
  77 #include <cam/cam_debug.h>
  78 #include <cam/cam_ccb.h>
  79 #include <cam/scsi/scsi_all.h>
  80 #include <cam/scsi/scsi_da.h>
  81 #include <cam/scsi/scsi_pass.h>
  82 #include <cam/scsi/scsi_message.h>
  83 #include <cam/scsi/smp_all.h>
  84 #include <camlib.h>
  85 #include <mtlib.h>
  86 #include <zlib.h>
  87
  88 typedef enum {
  89         CAMDD_CMD_NONE          = 0x00000000,
  90         CAMDD_CMD_HELP          = 0x00000001,
  91         CAMDD_CMD_WRITE         = 0x00000002,
  92         CAMDD_CMD_READ          = 0x00000003
  93 } camdd_cmdmask;
  94
  95 typedef enum {
  96         CAMDD_ARG_NONE          = 0x00000000,
  97         CAMDD_ARG_VERBOSE       = 0x00000001,
  98         CAMDD_ARG_DEVICE        = 0x00000002,
  99         CAMDD_ARG_BUS           = 0x00000004,
 100         CAMDD_ARG_TARGET        = 0x00000008,
 101         CAMDD_ARG_LUN           = 0x00000010,
 102         CAMDD_ARG_UNIT          = 0x00000020,
 103         CAMDD_ARG_TIMEOUT       = 0x00000040,
 104         CAMDD_ARG_ERR_RECOVER   = 0x00000080,
 105         CAMDD_ARG_RETRIES       = 0x00000100
 106 } camdd_argmask;
 107
 108 typedef enum {
 109         CAMDD_DEV_NONE          = 0x00,
 110         CAMDD_DEV_PASS          = 0x01,
 111         CAMDD_DEV_FILE          = 0x02
 112 } camdd_dev_type;
 113
 114 struct camdd_io_opts {
 115         camdd_dev_type  dev_type;
 116         char            *dev_name;
 117         uint64_t        blocksize;
 118         uint64_t        queue_depth;
 119         uint64_t        offset;
 120         int             min_cmd_size;
 121         int             write_dev;
 122         uint64_t        debug;
 123 };
 124
 125 typedef enum {
 126         CAMDD_BUF_NONE,
 127         CAMDD_BUF_DATA,
 128         CAMDD_BUF_INDIRECT
 129 } camdd_buf_type;
 130
 131 struct camdd_buf_indirect {
 132         /*
 133          * Pointer to the source buffer.
 134          */
 135         struct camdd_buf *src_buf;
 136
 137         /*
 138          * Offset into the source buffer, in bytes.
 139          */
 140         uint64_t          offset;
 141         /*
 142          * Pointer to the starting point in the source buffer.
 143          */
 144         uint8_t          *start_ptr;
 145
 146         /*
 147          * Length of this chunk in bytes.
 148          */
 149         size_t            len;
 150 };
 151
 152 struct camdd_buf_data {
 153         /*
 154          * Buffer allocated when we allocate this camdd_buf.  This should
 155          * be the size of the blocksize for this device.
 156          */
 157         uint8_t                 *buf;
 158
 159         /*
 160          * The amount of backing store allocated in buf.  Generally this
 161          * will be the blocksize of the device.
 162          */
 163         uint32_t                 alloc_len;
 164
 165         /*
 166          * The amount of data that was put into the buffer (on reads) or
 167          * the amount of data we have put onto the src_list so far (on
 168          * writes).
 169          */
 170         uint32_t                 fill_len;
 171
 172         /*
 173          * The amount of data that was not transferred.
 174          */
 175         uint32_t                 resid;
 176
 177         /*
 178          * Starting byte offset on the reader.
 179          */
 180         uint64_t                 src_start_offset;
 181
 182         /*
 183          * CCB used for pass(4) device targets.
 184          */
 185         union ccb                ccb;
 186
 187         /*
 188          * Number of scatter/gather segments.
 189          */
 190         int                      sg_count;
 191
 192         /*
 193          * Set if we had to tack on an extra buffer to round the transfer
 194          * up to a sector size.
 195          */
 196         int                      extra_buf;
 197
 198         /*
 199          * Scatter/gather list used generally when we're the writer for a
 200          * pass(4) device.
 201          */
 202         bus_dma_segment_t       *segs;
 203
 204         /*
 205          * Scatter/gather list used generally when we're the writer for a
 206          * file or block device;
 207          */
 208         struct iovec            *iovec;
 209 };
 210
 211 union camdd_buf_types {
 212         struct camdd_buf_indirect       indirect;
 213         struct camdd_buf_data           data;
 214 };
 215
 216 typedef enum {
 217         CAMDD_STATUS_NONE,
 218         CAMDD_STATUS_OK,
 219         CAMDD_STATUS_SHORT_IO,
 220         CAMDD_STATUS_EOF,
 221         CAMDD_STATUS_ERROR
 222 } camdd_buf_status;
 223
 224 struct camdd_buf {
 225         camdd_buf_type           buf_type;
 226         union camdd_buf_types    buf_type_spec;
 227
 228         camdd_buf_status         status;
 229
 230         uint64_t                 lba;
 231         size_t                   len;
 232
 233         /*
 234          * A reference count of how many indirect buffers point to this
 235          * buffer.
 236          */
 237         int                      refcount;
 238
 239         /*
 240          * A link back to our parent device.
 241          */
 242         struct camdd_dev        *dev;
 243         STAILQ_ENTRY(camdd_buf)  links;
 244         STAILQ_ENTRY(camdd_buf)  work_links;
 245
 246         /*
 247          * A count of the buffers on the src_list.
 248          */
 249         int                      src_count;
 250
 251         /*
 252          * List of buffers from our partner thread that are the components
 253          * of this buffer for the I/O.  Uses src_links.
 254          */
 255         STAILQ_HEAD(,camdd_buf)  src_list;
 256         STAILQ_ENTRY(camdd_buf)  src_links;
 257 };
 258
 259 #define NUM_DEV_TYPES   2
 260
 261 struct camdd_dev_pass {
 262         int                      scsi_dev_type;
 263         struct cam_device       *dev;
 264         uint64_t                 max_sector;
 265         uint32_t                 block_len;
 266         uint32_t                 cpi_maxio;
 267 };
 268
 269 typedef enum {
 270         CAMDD_FILE_NONE,
 271         CAMDD_FILE_REG,
 272         CAMDD_FILE_STD,
 273         CAMDD_FILE_PIPE,
 274         CAMDD_FILE_DISK,
 275         CAMDD_FILE_TAPE,
 276         CAMDD_FILE_TTY,
 277         CAMDD_FILE_MEM
 278 } camdd_file_type;
 279
 280 typedef enum {
 281         CAMDD_FF_NONE           = 0x00,
 282         CAMDD_FF_CAN_SEEK       = 0x01
 283 } camdd_file_flags;
 284
 285 struct camdd_dev_file {
 286         int                      fd;
 287         struct stat              sb;
 288         char                     filename[MAXPATHLEN + 1];
 289         camdd_file_type          file_type;
 290         camdd_file_flags         file_flags;
 291         uint8_t                 *tmp_buf;
 292 };
 293
 294 struct camdd_dev_block {
 295         int                      fd;
 296         uint64_t                 size_bytes;
 297         uint32_t                 block_len;
 298 };
 299
 300 union camdd_dev_spec {
 301         struct camdd_dev_pass   pass;
 302         struct camdd_dev_file   file;
 303         struct camdd_dev_block  block;
 304 };
 305
 306 typedef enum {
 307         CAMDD_DEV_FLAG_NONE             = 0x00,
 308         CAMDD_DEV_FLAG_EOF              = 0x01,
 309         CAMDD_DEV_FLAG_PEER_EOF         = 0x02,
 310         CAMDD_DEV_FLAG_ACTIVE           = 0x04,
 311         CAMDD_DEV_FLAG_EOF_SENT         = 0x08,
 312         CAMDD_DEV_FLAG_EOF_QUEUED       = 0x10
 313 } camdd_dev_flags;
 314
 315 struct camdd_dev {
 316         camdd_dev_type           dev_type;
 317         union camdd_dev_spec     dev_spec;
 318         camdd_dev_flags          flags;
 319         char                     device_name[MAXPATHLEN+1];
 320         uint32_t                 blocksize;
 321         uint32_t                 sector_size;
 322         uint64_t                 max_sector;
 323         uint64_t                 sector_io_limit;
 324         int                      min_cmd_size;
 325         int                      write_dev;
 326         int                      retry_count;
 327         int                      io_timeout;
 328         int                      debug;
 329         uint64_t                 start_offset_bytes;
 330         uint64_t                 next_io_pos_bytes;
 331         uint64_t                 next_peer_pos_bytes;
 332         uint64_t                 next_completion_pos_bytes;
 333         uint64_t                 peer_bytes_queued;
 334         uint64_t                 bytes_transferred;
 335         uint32_t                 target_queue_depth;
 336         uint32_t                 cur_active_io;
 337         uint8_t                 *extra_buf;
 338         uint32_t                 extra_buf_len;
 339         struct camdd_dev        *peer_dev;
 340         pthread_mutex_t          mutex;
 341         pthread_cond_t           cond;
 342         int                      kq;
 343
 344         int                      (*run)(struct camdd_dev *dev);
 345         int                      (*fetch)(struct camdd_dev *dev);
 346
 347         /*
 348          * Buffers that are available for I/O.  Uses links.
 349          */
 350         STAILQ_HEAD(,camdd_buf)  free_queue;
 351
 352         /*
 353          * Free indirect buffers.  These are used for breaking a large
 354          * buffer into multiple pieces.
 355          */
 356         STAILQ_HEAD(,camdd_buf)  free_indirect_queue;
 357
 358         /*
 359          * Buffers that have been queued to the kernel.  Uses links.
 360          */
 361         STAILQ_HEAD(,camdd_buf)  active_queue;
 362
 363         /*
 364          * Will generally contain one of our buffers that is waiting for enough
 365          * I/O from our partner thread to be able to execute.  This will
 366          * generally happen when our per-I/O-size is larger than the
 367          * partner thread's per-I/O-size.  Uses links.
 368          */
 369         STAILQ_HEAD(,camdd_buf)  pending_queue;
 370
 371         /*
 372          * Number of buffers on the pending queue
 373          */
 374         int                      num_pending_queue;
 375
 376         /*
 377          * Buffers that are filled and ready to execute.  This is used when
 378          * our partner (reader) thread sends us blocks that are larger than
 379          * our blocksize, and so we have to split them into multiple pieces.
 380          */
 381         STAILQ_HEAD(,camdd_buf)  run_queue;
 382
 383         /*
 384          * Number of buffers on the run queue.
 385          */
 386         int                      num_run_queue;
 387
 388         STAILQ_HEAD(,camdd_buf)  reorder_queue;
 389
 390         int                      num_reorder_queue;
 391
 392         /*
 393          * Buffers that have been queued to us by our partner thread
 394          * (generally the reader thread) to be written out.  Uses
 395          * work_links.
 396          */
 397         STAILQ_HEAD(,camdd_buf)  work_queue;
 398
 399         /*
 400          * Buffers that have been completed by our partner thread.  Uses
 401          * work_links.
 402          */
 403         STAILQ_HEAD(,camdd_buf)  peer_done_queue;
 404
 405         /*
 406          * Number of buffers on the peer done queue.
 407          */
 408         uint32_t                 num_peer_done_queue;
 409
 410         /*
 411          * A list of buffers that we have queued to our peer thread.  Uses
 412          * links.
 413          */
 414         STAILQ_HEAD(,camdd_buf)  peer_work_queue;
 415
 416         /*
 417          * Number of buffers on the peer work queue.
 418          */
 419         uint32_t                 num_peer_work_queue;
 420 };
 421
 422 static sem_t camdd_sem;
 423 static int need_exit = 0;
 424 static int error_exit = 0;
 425 static int need_status = 0;
 426
 427 #ifndef min
 428 #define min(a, b) (a < b) ? a : b
 429 #endif
 430
 431 /*
 432  * XXX KDM private copy of timespecsub().  This is normally defined in
 433  * sys/time.h, but is only enabled in the kernel.  If that definition is
 434  * enabled in userland, it breaks the build of libnetbsd.
 435  */
 436 #ifndef timespecsub
 437 #define timespecsub(vvp, uvp)                                           \
 438         do {                                                            \
 439                 (vvp)->tv_sec -= (uvp)->tv_sec;                         \
 440                 (vvp)->tv_nsec -= (uvp)->tv_nsec;                       \
 441                 if ((vvp)->tv_nsec < 0) {                               \
 442                         (vvp)->tv_sec--;                                \
 443                         (vvp)->tv_nsec += 1000000000;                   \
 444                 }                                                       \
 445         } while (0)
 446 #endif
 447
 448
 449 /* Generically usefull offsets into the peripheral private area */
 450 #define ppriv_ptr0 periph_priv.entries[0].ptr
 451 #define ppriv_ptr1 periph_priv.entries[1].ptr
 452 #define ppriv_field0 periph_priv.entries[0].field
 453 #define ppriv_field1 periph_priv.entries[1].field
 454
 455 #define ccb_buf ppriv_ptr0
 456
 457 #define CAMDD_FILE_DEFAULT_BLOCK        524288
 458 #define CAMDD_FILE_DEFAULT_DEPTH        1
 459 #define CAMDD_PASS_MAX_BLOCK            1048576
 460 #define CAMDD_PASS_DEFAULT_DEPTH        6
 461 #define CAMDD_PASS_RW_TIMEOUT           60 * 1000
 462
 463 static int parse_btl(char *tstr, int *bus, int *target, int *lun,
 464                      camdd_argmask *arglst);
 465 void camdd_free_dev(struct camdd_dev *dev);
 466 struct camdd_dev *camdd_alloc_dev(camdd_dev_type dev_type,
 467                                   struct kevent *new_ke, int num_ke,
 468                                   int retry_count, int timeout);
 469 static struct camdd_buf *camdd_alloc_buf(struct camdd_dev *dev,
 470                                          camdd_buf_type buf_type);
 471 void camdd_release_buf(struct camdd_buf *buf);
 472 struct camdd_buf *camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type);
 473 int camdd_buf_sg_create(struct camdd_buf *buf, int iovec,
 474                         uint32_t sector_size, uint32_t *num_sectors_used,
 475                         int *double_buf_needed);
 476 uint32_t camdd_buf_get_len(struct camdd_buf *buf);
 477 void camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf);
 478 int camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize,
 479                      uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran);
 480 struct camdd_dev *camdd_probe_file(int fd, struct camdd_io_opts *io_opts,
 481                                    int retry_count, int timeout);
 482 struct camdd_dev *camdd_probe_pass(struct cam_device *cam_dev,
 483                                    struct camdd_io_opts *io_opts,
 484                                    camdd_argmask arglist, int probe_retry_count,
 485                                    int probe_timeout, int io_retry_count,
 486                                    int io_timeout);
 487 void *camdd_file_worker(void *arg);
 488 camdd_buf_status camdd_ccb_status(union ccb *ccb);
 489 int camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf);
 490 int camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf);
 491 void camdd_peer_done(struct camdd_buf *buf);
 492 void camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf,
 493                         int *error_count);
 494 int camdd_pass_fetch(struct camdd_dev *dev);
 495 int camdd_file_run(struct camdd_dev *dev);
 496 int camdd_pass_run(struct camdd_dev *dev);
 497 int camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len);
 498 int camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf);
 499 void camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth,
 500                      uint32_t *peer_depth, uint32_t *our_bytes,
 501                      uint32_t *peer_bytes);
 502 void *camdd_worker(void *arg);
 503 void camdd_sig_handler(int sig);
 504 void camdd_print_status(struct camdd_dev *camdd_dev,
 505                         struct camdd_dev *other_dev,
 506                         struct timespec *start_time);
 507 int camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts,
 508              uint64_t max_io, int retry_count, int timeout);
 509 int camdd_parse_io_opts(char *args, int is_write,
 510                         struct camdd_io_opts *io_opts);
 511 void usage(void);
 512
 513 /*
 514  * Parse out a bus, or a bus, target and lun in the following
 515  * format:
 516  * bus
 517  * bus:target
 518  * bus:target:lun
 519  *
 520  * Returns the number of parsed components, or 0.
 521  */
 522 static int
 523 parse_btl(char *tstr, int *bus, int *target, int *lun, camdd_argmask *arglst)
 524 {
 525         char *tmpstr;
 526         int convs = 0;
 527
 528         while (isspace(*tstr) && (*tstr != '\0'))
 529                 tstr++;
 530
 531         tmpstr = (char *)strtok(tstr, ":");
 532         if ((tmpstr != NULL) && (*tmpstr != '\0')) {
 533                 *bus = strtol(tmpstr, NULL, 0);
 534                 *arglst |= CAMDD_ARG_BUS;
 535                 convs++;
 536                 tmpstr = (char *)strtok(NULL, ":");
 537                 if ((tmpstr != NULL) && (*tmpstr != '\0')) {
 538                         *target = strtol(tmpstr, NULL, 0);
 539                         *arglst |= CAMDD_ARG_TARGET;
 540                         convs++;
 541                         tmpstr = (char *)strtok(NULL, ":");
 542                         if ((tmpstr != NULL) && (*tmpstr != '\0')) {
 543                                 *lun = strtol(tmpstr, NULL, 0);
 544                                 *arglst |= CAMDD_ARG_LUN;
 545                                 convs++;
 546                         }
 547                 }
 548         }
 549
 550         return convs;
 551 }
 552
 553 /*
 554  * XXX KDM clean up and free all of the buffers on the queue!
 555  */
 556 void
 557 camdd_free_dev(struct camdd_dev *dev)
 558 {
 559         if (dev == NULL)
 560                 return;
 561
 562         switch (dev->dev_type) {
 563         case CAMDD_DEV_FILE: {
 564                 struct camdd_dev_file *file_dev = &dev->dev_spec.file;
 565
 566                 if (file_dev->fd != -1)
 567                         close(file_dev->fd);
 568                 free(file_dev->tmp_buf);
 569                 break;
 570         }
 571         case CAMDD_DEV_PASS: {
 572                 struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass;
 573
 574                 if (pass_dev->dev != NULL)
 575                         cam_close_device(pass_dev->dev);
 576                 break;
 577         }
 578         default:
 579                 break;
 580         }
 581
 582         free(dev);
 583 }
 584
 585 struct camdd_dev *
 586 camdd_alloc_dev(camdd_dev_type dev_type, struct kevent *new_ke, int num_ke,
 587                 int retry_count, int timeout)
 588 {
 589         struct camdd_dev *dev = NULL;
 590         struct kevent *ke;
 591         size_t ke_size;
 592         int retval = 0;
 593
 594         dev = malloc(sizeof(*dev));
 595         if (dev == NULL) {
 596                 warn("%s: unable to malloc %zu bytes", __func__, sizeof(*dev));
 597                 goto bailout;
 598         }
 599
 600         bzero(dev, sizeof(*dev));
 601
 602         dev->dev_type = dev_type;
 603         dev->io_timeout = timeout;
 604         dev->retry_count = retry_count;
 605         STAILQ_INIT(&dev->free_queue);
 606         STAILQ_INIT(&dev->free_indirect_queue);
 607         STAILQ_INIT(&dev->active_queue);
 608         STAILQ_INIT(&dev->pending_queue);
 609         STAILQ_INIT(&dev->run_queue);
 610         STAILQ_INIT(&dev->reorder_queue);
 611         STAILQ_INIT(&dev->work_queue);
 612         STAILQ_INIT(&dev->peer_done_queue);
 613         STAILQ_INIT(&dev->peer_work_queue);
 614         retval = pthread_mutex_init(&dev->mutex, NULL);
 615         if (retval != 0) {
 616                 warnc(retval, "%s: failed to initialize mutex", __func__);
 617                 goto bailout;
 618         }
 619
 620         retval = pthread_cond_init(&dev->cond, NULL);
 621         if (retval != 0) {
 622                 warnc(retval, "%s: failed to initialize condition variable",
 623                       __func__);
 624                 goto bailout;
 625         }
 626
 627         dev->kq = kqueue();
 628         if (dev->kq == -1) {
 629                 warn("%s: Unable to create kqueue", __func__);
 630                 goto bailout;
 631         }
 632
 633         ke_size = sizeof(struct kevent) * (num_ke + 4);
 634         ke = malloc(ke_size);
 635         if (ke == NULL) {
 636                 warn("%s: unable to malloc %zu bytes", __func__, ke_size);
 637                 goto bailout;
 638         }
 639         bzero(ke, ke_size);
 640         if (num_ke > 0)
 641                 bcopy(new_ke, ke, num_ke * sizeof(struct kevent));
 642
 643         EV_SET(&ke[num_ke++], (uintptr_t)&dev->work_queue, EVFILT_USER,
 644                EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0);
 645         EV_SET(&ke[num_ke++], (uintptr_t)&dev->peer_done_queue, EVFILT_USER,
 646                EV_ADD|EV_ENABLE|EV_CLEAR, 0,0, 0);
 647         EV_SET(&ke[num_ke++], SIGINFO, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0);
 648         EV_SET(&ke[num_ke++], SIGINT, EVFILT_SIGNAL, EV_ADD|EV_ENABLE, 0,0,0);
 649
 650         retval = kevent(dev->kq, ke, num_ke, NULL, 0, NULL);
 651         if (retval == -1) {
 652                 warn("%s: Unable to register kevents", __func__);
 653                 goto bailout;
 654         }
 655
 656
 657         return (dev);
 658
 659 bailout:
 660         free(dev);
 661
 662         return (NULL);
 663 }
 664
 665 static struct camdd_buf *
 666 camdd_alloc_buf(struct camdd_dev *dev, camdd_buf_type buf_type)
 667 {
 668         struct camdd_buf *buf = NULL;
 669         uint8_t *data_ptr = NULL;
 670
 671         /*
 672          * We only need to allocate data space for data buffers.
 673          */
 674         switch (buf_type) {
 675         case CAMDD_BUF_DATA:
 676                 data_ptr = malloc(dev->blocksize);
 677                 if (data_ptr == NULL) {
 678                         warn("unable to allocate %u bytes", dev->blocksize);
 679                         goto bailout_error;
 680                 }
 681                 break;
 682         default:
 683                 break;
 684         }
 685
 686         buf = malloc(sizeof(*buf));
 687         if (buf == NULL) {
 688                 warn("unable to allocate %zu bytes", sizeof(*buf));
 689                 goto bailout_error;
 690         }
 691
 692         bzero(buf, sizeof(*buf));
 693         buf->buf_type = buf_type;
 694         buf->dev = dev;
 695         switch (buf_type) {
 696         case CAMDD_BUF_DATA: {
 697                 struct camdd_buf_data *data;
 698
 699                 data = &buf->buf_type_spec.data;
 700
 701                 data->alloc_len = dev->blocksize;
 702                 data->buf = data_ptr;
 703                 break;
 704         }
 705         case CAMDD_BUF_INDIRECT:
 706                 break;
 707         default:
 708                 break;
 709         }
 710         STAILQ_INIT(&buf->src_list);
 711
 712         return (buf);
 713
 714 bailout_error:
 715         if (data_ptr != NULL)
 716                 free(data_ptr);
 717
 718         if (buf != NULL)
 719                 free(buf);
 720
 721         return (NULL);
 722 }
 723
 724 void
 725 camdd_release_buf(struct camdd_buf *buf)
 726 {
 727         struct camdd_dev *dev;
 728
 729         dev = buf->dev;
 730
 731         switch (buf->buf_type) {
 732         case CAMDD_BUF_DATA: {
 733                 struct camdd_buf_data *data;
 734
 735                 data = &buf->buf_type_spec.data;
 736
 737                 if (data->segs != NULL) {
 738                         if (data->extra_buf != 0) {
 739                                 void *extra_buf;
 740
 741                                 extra_buf = (void *)
 742                                     data->segs[data->sg_count - 1].ds_addr;
 743                                 free(extra_buf);
 744                                 data->extra_buf = 0;
 745                         }
 746                         free(data->segs);
 747                         data->segs = NULL;
 748                         data->sg_count = 0;
 749                 } else if (data->iovec != NULL) {
 750                         if (data->extra_buf != 0) {
 751                                 free(data->iovec[data->sg_count - 1].iov_base);
 752                                 data->extra_buf = 0;
 753                         }
 754                         free(data->iovec);
 755                         data->iovec = NULL;
 756                         data->sg_count = 0;
 757                 }
 758                 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
 759                 break;
 760         }
 761         case CAMDD_BUF_INDIRECT:
 762                 STAILQ_INSERT_TAIL(&dev->free_indirect_queue, buf, links);
 763                 break;
 764         default:
 765                 err(1, "%s: Invalid buffer type %d for released buffer",
 766                     __func__, buf->buf_type);
 767                 break;
 768         }
 769 }
 770
 771 struct camdd_buf *
 772 camdd_get_buf(struct camdd_dev *dev, camdd_buf_type buf_type)
 773 {
 774         struct camdd_buf *buf = NULL;
 775
 776         switch (buf_type) {
 777         case CAMDD_BUF_DATA:
 778                 buf = STAILQ_FIRST(&dev->free_queue);
 779                 if (buf != NULL) {
 780                         struct camdd_buf_data *data;
 781                         uint8_t *data_ptr;
 782                         uint32_t alloc_len;
 783
 784                         STAILQ_REMOVE_HEAD(&dev->free_queue, links);
 785                         data = &buf->buf_type_spec.data;
 786                         data_ptr = data->buf;
 787                         alloc_len = data->alloc_len;
 788                         bzero(buf, sizeof(*buf));
 789                         data->buf = data_ptr;
 790                         data->alloc_len = alloc_len;
 791                 }
 792                 break;
 793         case CAMDD_BUF_INDIRECT:
 794                 buf = STAILQ_FIRST(&dev->free_indirect_queue);
 795                 if (buf != NULL) {
 796                         STAILQ_REMOVE_HEAD(&dev->free_indirect_queue, links);
 797
 798                         bzero(buf, sizeof(*buf));
 799                 }
 800                 break;
 801         default:
 802                 warnx("Unknown buffer type %d requested", buf_type);
 803                 break;
 804         }
 805
 806
 807         if (buf == NULL)
 808                 return (camdd_alloc_buf(dev, buf_type));
 809         else {
 810                 STAILQ_INIT(&buf->src_list);
 811                 buf->dev = dev;
 812                 buf->buf_type = buf_type;
 813
 814                 return (buf);
 815         }
 816 }
 817
 818 int
 819 camdd_buf_sg_create(struct camdd_buf *buf, int iovec, uint32_t sector_size,
 820                     uint32_t *num_sectors_used, int *double_buf_needed)
 821 {
 822         struct camdd_buf *tmp_buf;
 823         struct camdd_buf_data *data;
 824         uint8_t *extra_buf = NULL;
 825         size_t extra_buf_len = 0;
 826         int i, retval = 0;
 827
 828         data = &buf->buf_type_spec.data;
 829
 830         data->sg_count = buf->src_count;
 831         /*
 832          * Compose a scatter/gather list from all of the buffers in the list.
 833          * If the length of the buffer isn't a multiple of the sector size,
 834          * we'll have to add an extra buffer.  This should only happen
 835          * at the end of a transfer.
 836          */
 837         if ((data->fill_len % sector_size) != 0) {
 838                 extra_buf_len = sector_size - (data->fill_len % sector_size);
 839                 extra_buf = calloc(extra_buf_len, 1);
 840                 if (extra_buf == NULL) {
 841                         warn("%s: unable to allocate %zu bytes for extra "
 842                             "buffer space", __func__, extra_buf_len);
 843                         retval = 1;
 844                         goto bailout;
 845                 }
 846                 data->extra_buf = 1;
 847                 data->sg_count++;
 848         }
 849         if (iovec == 0) {
 850                 data->segs = calloc(data->sg_count, sizeof(bus_dma_segment_t));
 851                 if (data->segs == NULL) {
 852                         warn("%s: unable to allocate %zu bytes for S/G list",
 853                             __func__, sizeof(bus_dma_segment_t) *
 854                             data->sg_count);
 855                         retval = 1;
 856                         goto bailout;
 857                 }
 858
 859         } else {
 860                 data->iovec = calloc(data->sg_count, sizeof(struct iovec));
 861                 if (data->iovec == NULL) {
 862                         warn("%s: unable to allocate %zu bytes for S/G list",
 863                             __func__, sizeof(struct iovec) * data->sg_count);
 864                         retval = 1;
 865                         goto bailout;
 866                 }
 867         }
 868
 869         for (i = 0, tmp_buf = STAILQ_FIRST(&buf->src_list);
 870              i < buf->src_count && tmp_buf != NULL; i++,
 871              tmp_buf = STAILQ_NEXT(tmp_buf, src_links)) {
 872
 873                 if (tmp_buf->buf_type == CAMDD_BUF_DATA) {
 874                         struct camdd_buf_data *tmp_data;
 875
 876                         tmp_data = &tmp_buf->buf_type_spec.data;
 877                         if (iovec == 0) {
 878                                 data->segs[i].ds_addr =
 879                                     (bus_addr_t) tmp_data->buf;
 880                                 data->segs[i].ds_len = tmp_data->fill_len -
 881                                     tmp_data->resid;
 882                         } else {
 883                                 data->iovec[i].iov_base = tmp_data->buf;
 884                                 data->iovec[i].iov_len = tmp_data->fill_len -
 885                                     tmp_data->resid;
 886                         }
 887                         if (((tmp_data->fill_len - tmp_data->resid) %
 888                              sector_size) != 0)
 889                                 *double_buf_needed = 1;
 890                 } else {
 891                         struct camdd_buf_indirect *tmp_ind;
 892
 893                         tmp_ind = &tmp_buf->buf_type_spec.indirect;
 894                         if (iovec == 0) {
 895                                 data->segs[i].ds_addr =
 896                                     (bus_addr_t)tmp_ind->start_ptr;
 897                                 data->segs[i].ds_len = tmp_ind->len;
 898                         } else {
 899                                 data->iovec[i].iov_base = tmp_ind->start_ptr;
 900                                 data->iovec[i].iov_len = tmp_ind->len;
 901                         }
 902                         if ((tmp_ind->len % sector_size) != 0)
 903                                 *double_buf_needed = 1;
 904                 }
 905         }
 906
 907         if (extra_buf != NULL) {
 908                 if (iovec == 0) {
 909                         data->segs[i].ds_addr = (bus_addr_t)extra_buf;
 910                         data->segs[i].ds_len = extra_buf_len;
 911                 } else {
 912                         data->iovec[i].iov_base = extra_buf;
 913                         data->iovec[i].iov_len = extra_buf_len;
 914                 }
 915                 i++;
 916         }
 917         if ((tmp_buf != NULL) || (i != data->sg_count)) {
 918                 warnx("buffer source count does not match "
 919                       "number of buffers in list!");
 920                 retval = 1;
 921                 goto bailout;
 922         }
 923
 924 bailout:
 925         if (retval == 0) {
 926                 *num_sectors_used = (data->fill_len + extra_buf_len) /
 927                     sector_size;
 928         }
 929         return (retval);
 930 }
 931
 932 uint32_t
 933 camdd_buf_get_len(struct camdd_buf *buf)
 934 {
 935         uint32_t len = 0;
 936
 937         if (buf->buf_type != CAMDD_BUF_DATA) {
 938                 struct camdd_buf_indirect *indirect;
 939
 940                 indirect = &buf->buf_type_spec.indirect;
 941                 len = indirect->len;
 942         } else {
 943                 struct camdd_buf_data *data;
 944
 945                 data = &buf->buf_type_spec.data;
 946                 len = data->fill_len;
 947         }
 948
 949         return (len);
 950 }
 951
 952 void
 953 camdd_buf_add_child(struct camdd_buf *buf, struct camdd_buf *child_buf)
 954 {
 955         struct camdd_buf_data *data;
 956
 957         assert(buf->buf_type == CAMDD_BUF_DATA);
 958
 959         data = &buf->buf_type_spec.data;
 960
 961         STAILQ_INSERT_TAIL(&buf->src_list, child_buf, src_links);
 962         buf->src_count++;
 963
 964         data->fill_len += camdd_buf_get_len(child_buf);
 965 }
 966
 967 typedef enum {
 968         CAMDD_TS_MAX_BLK,
 969         CAMDD_TS_MIN_BLK,
 970         CAMDD_TS_BLK_GRAN,
 971         CAMDD_TS_EFF_IOSIZE
 972 } camdd_status_item_index;
 973
 974 static struct camdd_status_items {
 975         const char *name;
 976         struct mt_status_entry *entry;
 977 } req_status_items[] = {
 978         { "max_blk", NULL },
 979         { "min_blk", NULL },
 980         { "blk_gran", NULL },
 981         { "max_effective_iosize", NULL }
 982 };
 983
 984 int
 985 camdd_probe_tape(int fd, char *filename, uint64_t *max_iosize,
 986                  uint64_t *max_blk, uint64_t *min_blk, uint64_t *blk_gran)
 987 {
 988         struct mt_status_data status_data;
 989         char *xml_str = NULL;
 990         unsigned int i;
 991         int retval = 0;
 992
 993         retval = mt_get_xml_str(fd, MTIOCEXTGET, &xml_str);
 994         if (retval != 0)
 995                 err(1, "Couldn't get XML string from %s", filename);
 996
 997         retval = mt_get_status(xml_str, &status_data);
 998         if (retval != XML_STATUS_OK) {
 999                 warn("couldn't get status for %s", filename);
1000                 retval = 1;
1001                 goto bailout;
1002         } else
1003                 retval = 0;
1004
1005         if (status_data.error != 0) {
1006                 warnx("%s", status_data.error_str);
1007                 retval = 1;
1008                 goto bailout;
1009         }
1010
1011         for (i = 0; i < sizeof(req_status_items) /
1012              sizeof(req_status_items[0]); i++) {
1013                 char *name;
1014
1015                 name = __DECONST(char *, req_status_items[i].name);
1016                 req_status_items[i].entry = mt_status_entry_find(&status_data,
1017                     name);
1018                 if (req_status_items[i].entry == NULL) {
1019                         errx(1, "Cannot find status entry %s",
1020                             req_status_items[i].name);
1021                 }
1022         }
1023
1024         *max_iosize = req_status_items[CAMDD_TS_EFF_IOSIZE].entry->value_unsigned;
1025         *max_blk= req_status_items[CAMDD_TS_MAX_BLK].entry->value_unsigned;
1026         *min_blk= req_status_items[CAMDD_TS_MIN_BLK].entry->value_unsigned;
1027         *blk_gran = req_status_items[CAMDD_TS_BLK_GRAN].entry->value_unsigned;
1028 bailout:
1029
1030         free(xml_str);
1031         mt_status_free(&status_data);
1032
1033         return (retval);
1034 }
1035
1036 struct camdd_dev *
1037 camdd_probe_file(int fd, struct camdd_io_opts *io_opts, int retry_count,
1038     int timeout)
1039 {
1040         struct camdd_dev *dev = NULL;
1041         struct camdd_dev_file *file_dev;
1042         uint64_t blocksize = io_opts->blocksize;
1043
1044         dev = camdd_alloc_dev(CAMDD_DEV_FILE, NULL, 0, retry_count, timeout);
1045         if (dev == NULL)
1046                 goto bailout;
1047
1048         file_dev = &dev->dev_spec.file;
1049         file_dev->fd = fd;
1050         strlcpy(file_dev->filename, io_opts->dev_name,
1051             sizeof(file_dev->filename));
1052         strlcpy(dev->device_name, io_opts->dev_name, sizeof(dev->device_name));
1053         if (blocksize == 0)
1054                 dev->blocksize = CAMDD_FILE_DEFAULT_BLOCK;
1055         else
1056                 dev->blocksize = blocksize;
1057
1058         if ((io_opts->queue_depth != 0)
1059          && (io_opts->queue_depth != 1)) {
1060                 warnx("Queue depth %ju for %s ignored, only 1 outstanding "
1061                     "command supported", (uintmax_t)io_opts->queue_depth,
1062                     io_opts->dev_name);
1063         }
1064         dev->target_queue_depth = CAMDD_FILE_DEFAULT_DEPTH;
1065         dev->run = camdd_file_run;
1066         dev->fetch = NULL;
1067
1068         /*
1069          * We can effectively access files on byte boundaries.  We'll reset
1070          * this for devices like disks that can be accessed on sector
1071          * boundaries.
1072          */
1073         dev->sector_size = 1;
1074
1075         if ((fd != STDIN_FILENO)
1076          && (fd != STDOUT_FILENO)) {
1077                 int retval;
1078
1079                 retval = fstat(fd, &file_dev->sb);
1080                 if (retval != 0) {
1081                         warn("Cannot stat %s", dev->device_name);
1082                         goto bailout;
1083                         camdd_free_dev(dev);
1084                         dev = NULL;
1085                 }
1086                 if (S_ISREG(file_dev->sb.st_mode)) {
1087                         file_dev->file_type = CAMDD_FILE_REG;
1088                 } else if (S_ISCHR(file_dev->sb.st_mode)) {
1089                         int type;
1090
1091                         if (ioctl(fd, FIODTYPE, &type) == -1)
1092                                 err(1, "FIODTYPE ioctl failed on %s",
1093                                     dev->device_name);
1094                         else {
1095                                 if (type & D_TAPE)
1096                                         file_dev->file_type = CAMDD_FILE_TAPE;
1097                                 else if (type & D_DISK)
1098                                         file_dev->file_type = CAMDD_FILE_DISK;
1099                                 else if (type & D_MEM)
1100                                         file_dev->file_type = CAMDD_FILE_MEM;
1101                                 else if (type & D_TTY)
1102                                         file_dev->file_type = CAMDD_FILE_TTY;
1103                         }
1104                 } else if (S_ISDIR(file_dev->sb.st_mode)) {
1105                         errx(1, "cannot operate on directory %s",
1106                             dev->device_name);
1107                 } else if (S_ISFIFO(file_dev->sb.st_mode)) {
1108                         file_dev->file_type = CAMDD_FILE_PIPE;
1109                 } else
1110                         errx(1, "Cannot determine file type for %s",
1111                             dev->device_name);
1112
1113                 switch (file_dev->file_type) {
1114                 case CAMDD_FILE_REG:
1115                         if (file_dev->sb.st_size != 0)
1116                                 dev->max_sector = file_dev->sb.st_size - 1;
1117                         else
1118                                 dev->max_sector = 0;
1119                         file_dev->file_flags |= CAMDD_FF_CAN_SEEK;
1120                         break;
1121                 case CAMDD_FILE_TAPE: {
1122                         uint64_t max_iosize, max_blk, min_blk, blk_gran;
1123                         /*
1124                          * Check block limits and maximum effective iosize.
1125                          * Make sure the blocksize is within the block
1126                          * limits (and a multiple of the minimum blocksize)
1127                          * and that the blocksize is <= maximum effective
1128                          * iosize.
1129                          */
1130                         retval = camdd_probe_tape(fd, dev->device_name,
1131                             &max_iosize, &max_blk, &min_blk, &blk_gran);
1132                         if (retval != 0)
1133                                 errx(1, "Unable to probe tape %s",
1134                                     dev->device_name);
1135
1136                         /*
1137                          * The blocksize needs to be <= the maximum
1138                          * effective I/O size of the tape device.  Note
1139                          * that this also takes into account the maximum
1140                          * blocksize reported by READ BLOCK LIMITS.
1141                          */
1142                         if (dev->blocksize > max_iosize) {
1143                                 warnx("Blocksize %u too big for %s, limiting "
1144                                     "to %ju", dev->blocksize, dev->device_name,
1145                                     max_iosize);
1146                                 dev->blocksize = max_iosize;
1147                         }
1148
1149                         /*
1150                          * The blocksize needs to be at least min_blk;
1151                          */
1152                         if (dev->blocksize < min_blk) {
1153                                 warnx("Blocksize %u too small for %s, "
1154                                     "increasing to %ju", dev->blocksize,
1155                                     dev->device_name, min_blk);
1156                                 dev->blocksize = min_blk;
1157                         }
1158
1159                         /*
1160                          * And the blocksize needs to be a multiple of
1161                          * the block granularity.
1162                          */
1163                         if ((blk_gran != 0)
1164                          && (dev->blocksize % (1 << blk_gran))) {
1165                                 warnx("Blocksize %u for %s not a multiple of "
1166                                     "%d, adjusting to %d", dev->blocksize,
1167                                     dev->device_name, (1 << blk_gran),
1168                                     dev->blocksize & ~((1 << blk_gran) - 1));
1169                                 dev->blocksize &= ~((1 << blk_gran) - 1);
1170                         }
1171
1172                         if (dev->blocksize == 0) {
1173                                 errx(1, "Unable to derive valid blocksize for "
1174                                     "%s", dev->device_name);
1175                         }
1176
1177                         /*
1178                          * For tape drives, set the sector size to the
1179                          * blocksize so that we make sure not to write
1180                          * less than the blocksize out to the drive.
1181                          */
1182                         dev->sector_size = dev->blocksize;
1183                         break;
1184                 }
1185                 case CAMDD_FILE_DISK: {
1186                         off_t media_size;
1187                         unsigned int sector_size;
1188
1189                         file_dev->file_flags |= CAMDD_FF_CAN_SEEK;
1190
1191                         if (ioctl(fd, DIOCGSECTORSIZE, &sector_size) == -1) {
1192                                 err(1, "DIOCGSECTORSIZE ioctl failed on %s",
1193                                     dev->device_name);
1194                         }
1195
1196                         if (sector_size == 0) {
1197                                 errx(1, "DIOCGSECTORSIZE ioctl returned "
1198                                     "invalid sector size %u for %s",
1199                                     sector_size, dev->device_name);
1200                         }
1201
1202                         if (ioctl(fd, DIOCGMEDIASIZE, &media_size) == -1) {
1203                                 err(1, "DIOCGMEDIASIZE ioctl failed on %s",
1204                                     dev->device_name);
1205                         }
1206
1207                         if (media_size == 0) {
1208                                 errx(1, "DIOCGMEDIASIZE ioctl returned "
1209                                     "invalid media size %ju for %s",
1210                                     (uintmax_t)media_size, dev->device_name);
1211                         }
1212
1213                         if (dev->blocksize % sector_size) {
1214                                 errx(1, "%s blocksize %u not a multiple of "
1215                                     "sector size %u", dev->device_name,
1216                                     dev->blocksize, sector_size);
1217                         }
1218
1219                         dev->sector_size = sector_size;
1220                         dev->max_sector = (media_size / sector_size) - 1;
1221                         break;
1222                 }
1223                 case CAMDD_FILE_MEM:
1224                         file_dev->file_flags |= CAMDD_FF_CAN_SEEK;
1225                         break;
1226                 default:
1227                         break;
1228                 }
1229         }
1230
1231         if ((io_opts->offset != 0)
1232          && ((file_dev->file_flags & CAMDD_FF_CAN_SEEK) == 0)) {
1233                 warnx("Offset %ju specified for %s, but we cannot seek on %s",
1234                     io_opts->offset, io_opts->dev_name, io_opts->dev_name);
1235                 goto bailout_error;
1236         }
1237 #if 0
1238         else if ((io_opts->offset != 0)
1239                 && ((io_opts->offset % dev->sector_size) != 0)) {
1240                 warnx("Offset %ju for %s is not a multiple of the "
1241                       "sector size %u", io_opts->offset,
1242                       io_opts->dev_name, dev->sector_size);
1243                 goto bailout_error;
1244         } else {
1245                 dev->start_offset_bytes = io_opts->offset;
1246         }
1247 #endif
1248
1249 bailout:
1250         return (dev);
1251
1252 bailout_error:
1253         camdd_free_dev(dev);
1254         return (NULL);
1255 }
1256
1257 /*
1258  * Need to implement this.  Do a basic probe:
1259  * - Check the inquiry data, make sure we're talking to a device that we
1260  *   can reasonably expect to talk to -- direct, RBC, CD, WORM.
1261  * - Send a test unit ready, make sure the device is available.
1262  * - Get the capacity and block size.
1263  */
1264 struct camdd_dev *
1265 camdd_probe_pass(struct cam_device *cam_dev, struct camdd_io_opts *io_opts,
1266                  camdd_argmask arglist, int probe_retry_count,
1267                  int probe_timeout, int io_retry_count, int io_timeout)
1268 {
1269         union ccb *ccb;
1270         uint64_t maxsector;
1271         uint32_t cpi_maxio, max_iosize, pass_numblocks;
1272         uint32_t block_len;
1273         struct scsi_read_capacity_data rcap;
1274         struct scsi_read_capacity_data_long rcaplong;
1275         struct camdd_dev *dev;
1276         struct camdd_dev_pass *pass_dev;
1277         struct kevent ke;
1278         int scsi_dev_type;
1279
1280         dev = NULL;
1281
1282         scsi_dev_type = SID_TYPE(&cam_dev->inq_data);
1283         maxsector = 0;
1284         block_len = 0;
1285
1286         /*
1287          * For devices that support READ CAPACITY, we'll attempt to get the
1288          * capacity.  Otherwise, we really don't support tape or other
1289          * devices via SCSI passthrough, so just return an error in that case.
1290          */
1291         switch (scsi_dev_type) {
1292         case T_DIRECT:
1293         case T_WORM:
1294         case T_CDROM:
1295         case T_OPTICAL:
1296         case T_RBC:
1297                 break;
1298         default:
1299                 errx(1, "Unsupported SCSI device type %d", scsi_dev_type);
1300                 break; /*NOTREACHED*/
1301         }
1302
1303         ccb = cam_getccb(cam_dev);
1304
1305         if (ccb == NULL) {
1306                 warnx("%s: error allocating ccb", __func__);
1307                 goto bailout;
1308         }
1309
1310         bzero(&(&ccb->ccb_h)[1],
1311               sizeof(struct ccb_scsiio) - sizeof(struct ccb_hdr));
1312
1313         scsi_read_capacity(&ccb->csio,
1314                            /*retries*/ probe_retry_count,
1315                            /*cbfcnp*/ NULL,
1316                            /*tag_action*/ MSG_SIMPLE_Q_TAG,
1317                            &rcap,
1318                            SSD_FULL_SIZE,
1319                            /*timeout*/ probe_timeout ? probe_timeout : 5000);
1320
1321         /* Disable freezing the device queue */
1322         ccb->ccb_h.flags |= CAM_DEV_QFRZDIS;
1323
1324         if (arglist & CAMDD_ARG_ERR_RECOVER)
1325                 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
1326
1327         if (cam_send_ccb(cam_dev, ccb) < 0) {
1328                 warn("error sending READ CAPACITY command");
1329
1330                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL,
1331                                 CAM_EPF_ALL, stderr);
1332
1333                 goto bailout;
1334         }
1335
1336         if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1337                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr);
1338                 goto bailout;
1339         }
1340
1341         maxsector = scsi_4btoul(rcap.addr);
1342         block_len = scsi_4btoul(rcap.length);
1343
1344         /*
1345          * A last block of 2^32-1 means that the true capacity is over 2TB,
1346          * and we need to issue the long READ CAPACITY to get the real
1347          * capacity.  Otherwise, we're all set.
1348          */
1349         if (maxsector != 0xffffffff)
1350                 goto rcap_done;
1351
1352         scsi_read_capacity_16(&ccb->csio,
1353                               /*retries*/ probe_retry_count,
1354                               /*cbfcnp*/ NULL,
1355                               /*tag_action*/ MSG_SIMPLE_Q_TAG,
1356                               /*lba*/ 0,
1357                               /*reladdr*/ 0,
1358                               /*pmi*/ 0,
1359                               (uint8_t *)&rcaplong,
1360                               sizeof(rcaplong),
1361                               /*sense_len*/ SSD_FULL_SIZE,
1362                               /*timeout*/ probe_timeout ? probe_timeout : 5000);
1363
1364         /* Disable freezing the device queue */
1365         ccb->ccb_h.flags |= CAM_DEV_QFRZDIS;
1366
1367         if (arglist & CAMDD_ARG_ERR_RECOVER)
1368                 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
1369
1370         if (cam_send_ccb(cam_dev, ccb) < 0) {
1371                 warn("error sending READ CAPACITY (16) command");
1372                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL,
1373                                 CAM_EPF_ALL, stderr);
1374                 goto bailout;
1375         }
1376
1377         if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1378                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL, CAM_EPF_ALL, stderr);
1379                 goto bailout;
1380         }
1381
1382         maxsector = scsi_8btou64(rcaplong.addr);
1383         block_len = scsi_4btoul(rcaplong.length);
1384
1385 rcap_done:
1386
1387         bzero(&(&ccb->ccb_h)[1],
1388               sizeof(struct ccb_scsiio) - sizeof(struct ccb_hdr));
1389
1390         ccb->ccb_h.func_code = XPT_PATH_INQ;
1391         ccb->ccb_h.flags = CAM_DIR_NONE;
1392         ccb->ccb_h.retry_count = 1;
1393
1394         if (cam_send_ccb(cam_dev, ccb) < 0) {
1395                 warn("error sending XPT_PATH_INQ CCB");
1396
1397                 cam_error_print(cam_dev, ccb, CAM_ESF_ALL,
1398                                 CAM_EPF_ALL, stderr);
1399                 goto bailout;
1400         }
1401
1402         EV_SET(&ke, cam_dev->fd, EVFILT_READ, EV_ADD|EV_ENABLE, 0, 0, 0);
1403
1404         dev = camdd_alloc_dev(CAMDD_DEV_PASS, &ke, 1, io_retry_count,
1405                               io_timeout);
1406         if (dev == NULL)
1407                 goto bailout;
1408
1409         pass_dev = &dev->dev_spec.pass;
1410         pass_dev->scsi_dev_type = scsi_dev_type;
1411         pass_dev->dev = cam_dev;
1412         pass_dev->max_sector = maxsector;
1413         pass_dev->block_len = block_len;
1414         pass_dev->cpi_maxio = ccb->cpi.maxio;
1415         snprintf(dev->device_name, sizeof(dev->device_name), "%s%u",
1416                  pass_dev->dev->device_name, pass_dev->dev->dev_unit_num);
1417         dev->sector_size = block_len;
1418         dev->max_sector = maxsector;
1419
1420
1421         /*
1422          * Determine the optimal blocksize to use for this device.
1423          */
1424
1425         /*
1426          * If the controller has not specified a maximum I/O size,
1427          * just go with 128K as a somewhat conservative value.
1428          */
1429         if (pass_dev->cpi_maxio == 0)
1430                 cpi_maxio = 131072;
1431         else
1432                 cpi_maxio = pass_dev->cpi_maxio;
1433
1434         /*
1435          * If the controller has a large maximum I/O size, limit it
1436          * to something smaller so that the kernel doesn't have trouble
1437          * allocating buffers to copy data in and out for us.
1438          * XXX KDM this is until we have unmapped I/O support in the kernel.
1439          */
1440         max_iosize = min(cpi_maxio, CAMDD_PASS_MAX_BLOCK);
1441
1442         /*
1443          * If we weren't able to get a block size for some reason,
1444          * default to 512 bytes.
1445          */
1446         block_len = pass_dev->block_len;
1447         if (block_len == 0)
1448                 block_len = 512;
1449
1450         /*
1451          * Figure out how many blocksize chunks will fit in the
1452          * maximum I/O size.
1453          */
1454         pass_numblocks = max_iosize / block_len;
1455
1456         /*
1457          * And finally, multiple the number of blocks by the LBA
1458          * length to get our maximum block size;
1459          */
1460         dev->blocksize = pass_numblocks * block_len;
1461
1462         if (io_opts->blocksize != 0) {
1463                 if ((io_opts->blocksize % dev->sector_size) != 0) {
1464                         warnx("Blocksize %ju for %s is not a multiple of "
1465                               "sector size %u", (uintmax_t)io_opts->blocksize,
1466                               dev->device_name, dev->sector_size);
1467                         goto bailout_error;
1468                 }
1469                 dev->blocksize = io_opts->blocksize;
1470         }
1471         dev->target_queue_depth = CAMDD_PASS_DEFAULT_DEPTH;
1472         if (io_opts->queue_depth != 0)
1473                 dev->target_queue_depth = io_opts->queue_depth;
1474
1475         if (io_opts->offset != 0) {
1476                 if (io_opts->offset > (dev->max_sector * dev->sector_size)) {
1477                         warnx("Offset %ju is past the end of device %s",
1478                             io_opts->offset, dev->device_name);
1479                         goto bailout_error;
1480                 }
1481 #if 0
1482                 else if ((io_opts->offset % dev->sector_size) != 0) {
1483                         warnx("Offset %ju for %s is not a multiple of the "
1484                               "sector size %u", io_opts->offset,
1485                               dev->device_name, dev->sector_size);
1486                         goto bailout_error;
1487                 }
1488                 dev->start_offset_bytes = io_opts->offset;
1489 #endif
1490         }
1491
1492         dev->min_cmd_size = io_opts->min_cmd_size;
1493
1494         dev->run = camdd_pass_run;
1495         dev->fetch = camdd_pass_fetch;
1496
1497 bailout:
1498         cam_freeccb(ccb);
1499
1500         return (dev);
1501
1502 bailout_error:
1503         cam_freeccb(ccb);
1504
1505         camdd_free_dev(dev);
1506
1507         return (NULL);
1508 }
1509
1510 void *
1511 camdd_worker(void *arg)
1512 {
1513         struct camdd_dev *dev = arg;
1514         struct camdd_buf *buf;
1515         struct timespec ts, *kq_ts;
1516
1517         ts.tv_sec = 0;
1518         ts.tv_nsec = 0;
1519
1520         pthread_mutex_lock(&dev->mutex);
1521
1522         dev->flags |= CAMDD_DEV_FLAG_ACTIVE;
1523
1524         for (;;) {
1525                 struct kevent ke;
1526                 int retval = 0;
1527
1528                 /*
1529                  * XXX KDM check the reorder queue depth?
1530                  */
1531                 if (dev->write_dev == 0) {
1532                         uint32_t our_depth, peer_depth, peer_bytes, our_bytes;
1533                         uint32_t target_depth = dev->target_queue_depth;
1534                         uint32_t peer_target_depth =
1535                             dev->peer_dev->target_queue_depth;
1536                         uint32_t peer_blocksize = dev->peer_dev->blocksize;
1537
1538                         camdd_get_depth(dev, &our_depth, &peer_depth,
1539                                         &our_bytes, &peer_bytes);
1540
1541 #if 0
1542                         while (((our_depth < target_depth)
1543                              && (peer_depth < peer_target_depth))
1544                             || ((peer_bytes + our_bytes) <
1545                                  (peer_blocksize * 2))) {
1546 #endif
1547                         while (((our_depth + peer_depth) <
1548                                 (target_depth + peer_target_depth))
1549                             || ((peer_bytes + our_bytes) <
1550                                 (peer_blocksize * 3))) {
1551
1552                                 retval = camdd_queue(dev, NULL);
1553                                 if (retval == 1)
1554                                         break;
1555                                 else if (retval != 0) {
1556                                         error_exit = 1;
1557                                         goto bailout;
1558                                 }
1559
1560                                 camdd_get_depth(dev, &our_depth, &peer_depth,
1561                                                 &our_bytes, &peer_bytes);
1562                         }
1563                 }
1564                 /*
1565                  * See if we have any I/O that is ready to execute.
1566                  */
1567                 buf = STAILQ_FIRST(&dev->run_queue);
1568                 if (buf != NULL) {
1569                         while (dev->target_queue_depth > dev->cur_active_io) {
1570                                 retval = dev->run(dev);
1571                                 if (retval == -1) {
1572                                         dev->flags |= CAMDD_DEV_FLAG_EOF;
1573                                         error_exit = 1;
1574                                         break;
1575                                 } else if (retval != 0) {
1576                                         break;
1577                                 }
1578                         }
1579                 }
1580
1581                 /*
1582                  * We've reached EOF, or our partner has reached EOF.
1583                  */
1584                 if ((dev->flags & CAMDD_DEV_FLAG_EOF)
1585                  || (dev->flags & CAMDD_DEV_FLAG_PEER_EOF)) {
1586                         if (dev->write_dev != 0) {
1587                                 if ((STAILQ_EMPTY(&dev->work_queue))
1588                                  && (dev->num_run_queue == 0)
1589                                  && (dev->cur_active_io == 0)) {
1590                                         goto bailout;
1591                                 }
1592                         } else {
1593                                 /*
1594                                  * If we're the reader, and the writer
1595                                  * got EOF, he is already done.  If we got
1596                                  * the EOF, then we need to wait until
1597                                  * everything is flushed out for the writer.
1598                                  */
1599                                 if (dev->flags & CAMDD_DEV_FLAG_PEER_EOF) {
1600                                         goto bailout;
1601                                 } else if ((dev->num_peer_work_queue == 0)
1602                                         && (dev->num_peer_done_queue == 0)
1603                                         && (dev->cur_active_io == 0)
1604                                         && (dev->num_run_queue == 0)) {
1605                                         goto bailout;
1606                                 }
1607                         }
1608                         /*
1609                          * XXX KDM need to do something about the pending
1610                          * queue and cleanup resources.
1611                          */
1612                 }
1613
1614                 if ((dev->write_dev == 0)
1615                  && (dev->cur_active_io == 0)
1616                  && (dev->peer_bytes_queued < dev->peer_dev->blocksize))
1617                         kq_ts = &ts;
1618                 else
1619                         kq_ts = NULL;
1620
1621                 /*
1622                  * Run kevent to see if there are events to process.
1623                  */
1624                 pthread_mutex_unlock(&dev->mutex);
1625                 retval = kevent(dev->kq, NULL, 0, &ke, 1, kq_ts);
1626                 pthread_mutex_lock(&dev->mutex);
1627                 if (retval == -1) {
1628                         warn("%s: error returned from kevent",__func__);
1629                         goto bailout;
1630                 } else if (retval != 0) {
1631                         switch (ke.filter) {
1632                         case EVFILT_READ:
1633                                 if (dev->fetch != NULL) {
1634                                         retval = dev->fetch(dev);
1635                                         if (retval == -1) {
1636                                                 error_exit = 1;
1637                                                 goto bailout;
1638                                         }
1639                                 }
1640                                 break;
1641                         case EVFILT_SIGNAL:
1642                                 /*
1643                                  * We register for this so we don't get
1644                                  * an error as a result of a SIGINFO or a
1645                                  * SIGINT.  It will actually get handled
1646                                  * by the signal handler.  If we get a
1647                                  * SIGINT, bail out without printing an
1648                                  * error message.  Any other signals
1649                                  * will result in the error message above.
1650                                  */
1651                                 if (ke.ident == SIGINT)
1652                                         goto bailout;
1653                                 break;
1654                         case EVFILT_USER:
1655                                 retval = 0;
1656                                 /*
1657                                  * Check to see if the other thread has
1658                                  * queued any I/O for us to do.  (In this
1659                                  * case we're the writer.)
1660                                  */
1661                                 for (buf = STAILQ_FIRST(&dev->work_queue);
1662                                      buf != NULL;
1663                                      buf = STAILQ_FIRST(&dev->work_queue)) {
1664                                         STAILQ_REMOVE_HEAD(&dev->work_queue,
1665                                                            work_links);
1666                                         retval = camdd_queue(dev, buf);
1667                                         /*
1668                                          * We keep going unless we get an
1669                                          * actual error.  If we get EOF, we
1670                                          * still want to remove the buffers
1671                                          * from the queue and send the back
1672                                          * to the reader thread.
1673                                          */
1674                                         if (retval == -1) {
1675                                                 error_exit = 1;
1676                                                 goto bailout;
1677                                         } else
1678                                                 retval = 0;
1679                                 }
1680
1681                                 /*
1682                                  * Next check to see if the other thread has
1683                                  * queued any completed buffers back to us.
1684                                  * (In this case we're the reader.)
1685                                  */
1686                                 for (buf = STAILQ_FIRST(&dev->peer_done_queue);
1687                                      buf != NULL;
1688                                      buf = STAILQ_FIRST(&dev->peer_done_queue)){
1689                                         STAILQ_REMOVE_HEAD(
1690                                             &dev->peer_done_queue, work_links);
1691                                         dev->num_peer_done_queue--;
1692                                         camdd_peer_done(buf);
1693                                 }
1694                                 break;
1695                         default:
1696                                 warnx("%s: unknown kevent filter %d",
1697                                       __func__, ke.filter);
1698                                 break;
1699                         }
1700                 }
1701         }
1702
1703 bailout:
1704
1705         dev->flags &= ~CAMDD_DEV_FLAG_ACTIVE;
1706
1707         /* XXX KDM cleanup resources here? */
1708
1709         pthread_mutex_unlock(&dev->mutex);
1710
1711         need_exit = 1;
1712         sem_post(&camdd_sem);
1713
1714         return (NULL);
1715 }
1716
1717 /*
1718  * Simplistic translation of CCB status to our local status.
1719  */
1720 camdd_buf_status
1721 camdd_ccb_status(union ccb *ccb)
1722 {
1723         camdd_buf_status status = CAMDD_STATUS_NONE;
1724         cam_status ccb_status;
1725
1726         ccb_status = ccb->ccb_h.status & CAM_STATUS_MASK;
1727
1728         switch (ccb_status) {
1729         case CAM_REQ_CMP: {
1730                 if (ccb->csio.resid == 0) {
1731                         status = CAMDD_STATUS_OK;
1732                 } else if (ccb->csio.dxfer_len > ccb->csio.resid) {
1733                         status = CAMDD_STATUS_SHORT_IO;
1734                 } else {
1735                         status = CAMDD_STATUS_EOF;
1736                 }
1737                 break;
1738         }
1739         case CAM_SCSI_STATUS_ERROR: {
1740                 switch (ccb->csio.scsi_status) {
1741                 case SCSI_STATUS_OK:
1742                 case SCSI_STATUS_COND_MET:
1743                 case SCSI_STATUS_INTERMED:
1744                 case SCSI_STATUS_INTERMED_COND_MET:
1745                         status = CAMDD_STATUS_OK;
1746                         break;
1747                 case SCSI_STATUS_CMD_TERMINATED:
1748                 case SCSI_STATUS_CHECK_COND:
1749                 case SCSI_STATUS_QUEUE_FULL:
1750                 case SCSI_STATUS_BUSY:
1751                 case SCSI_STATUS_RESERV_CONFLICT:
1752                 default:
1753                         status = CAMDD_STATUS_ERROR;
1754                         break;
1755                 }
1756                 break;
1757         }
1758         default:
1759                 status = CAMDD_STATUS_ERROR;
1760                 break;
1761         }
1762
1763         return (status);
1764 }
1765
1766 /*
1767  * Queue a buffer to our peer's work thread for writing.
1768  *
1769  * Returns 0 for success, -1 for failure, 1 if the other thread exited.
1770  */
1771 int
1772 camdd_queue_peer_buf(struct camdd_dev *dev, struct camdd_buf *buf)
1773 {
1774         struct kevent ke;
1775         STAILQ_HEAD(, camdd_buf) local_queue;
1776         struct camdd_buf *buf1, *buf2;
1777         struct camdd_buf_data *data = NULL;
1778         uint64_t peer_bytes_queued = 0;
1779         int active = 1;
1780         int retval = 0;
1781
1782         STAILQ_INIT(&local_queue);
1783
1784         /*
1785          * Since we're the reader, we need to queue our I/O to the writer
1786          * in sequential order in order to make sure it gets written out
1787          * in sequential order.
1788          *
1789          * Check the next expected I/O starting offset.  If this doesn't
1790          * match, put it on the reorder queue.
1791          */
1792         if ((buf->lba * dev->sector_size) != dev->next_completion_pos_bytes) {
1793
1794                 /*
1795                  * If there is nothing on the queue, there is no sorting
1796                  * needed.
1797                  */
1798                 if (STAILQ_EMPTY(&dev->reorder_queue)) {
1799                         STAILQ_INSERT_TAIL(&dev->reorder_queue, buf, links);
1800                         dev->num_reorder_queue++;
1801                         goto bailout;
1802                 }
1803
1804                 /*
1805                  * Sort in ascending order by starting LBA.  There should
1806                  * be no identical LBAs.
1807                  */
1808                 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL;
1809                      buf1 = buf2) {
1810                         buf2 = STAILQ_NEXT(buf1, links);
1811                         if (buf->lba < buf1->lba) {
1812                                 /*
1813                                  * If we're less than the first one, then
1814                                  * we insert at the head of the list
1815                                  * because this has to be the first element
1816                                  * on the list.
1817                                  */
1818                                 STAILQ_INSERT_HEAD(&dev->reorder_queue,
1819                                                    buf, links);
1820                                 dev->num_reorder_queue++;
1821                                 break;
1822                         } else if (buf->lba > buf1->lba) {
1823                                 if (buf2 == NULL) {
1824                                         STAILQ_INSERT_TAIL(&dev->reorder_queue,
1825                                             buf, links);
1826                                         dev->num_reorder_queue++;
1827                                         break;
1828                                 } else if (buf->lba < buf2->lba) {
1829                                         STAILQ_INSERT_AFTER(&dev->reorder_queue,
1830                                             buf1, buf, links);
1831                                         dev->num_reorder_queue++;
1832                                         break;
1833                                 }
1834                         } else {
1835                                 errx(1, "Found buffers with duplicate LBA %ju!",
1836                                      buf->lba);
1837                         }
1838                 }
1839                 goto bailout;
1840         } else {
1841
1842                 /*
1843                  * We're the next expected I/O completion, so put ourselves
1844                  * on the local queue to be sent to the writer.  We use
1845                  * work_links here so that we can queue this to the
1846                  * peer_work_queue before taking the buffer off of the
1847                  * local_queue.
1848                  */
1849                 dev->next_completion_pos_bytes += buf->len;
1850                 STAILQ_INSERT_TAIL(&local_queue, buf, work_links);
1851
1852                 /*
1853                  * Go through the reorder queue looking for more sequential
1854                  * I/O and add it to the local queue.
1855                  */
1856                 for (buf1 = STAILQ_FIRST(&dev->reorder_queue); buf1 != NULL;
1857                      buf1 = STAILQ_FIRST(&dev->reorder_queue)) {
1858                         /*
1859                          * As soon as we see an I/O that is out of sequence,
1860                          * we're done.
1861                          */
1862                         if ((buf1->lba * dev->sector_size) !=
1863                              dev->next_completion_pos_bytes)
1864                                 break;
1865
1866                         STAILQ_REMOVE_HEAD(&dev->reorder_queue, links);
1867                         dev->num_reorder_queue--;
1868                         STAILQ_INSERT_TAIL(&local_queue, buf1, work_links);
1869                         dev->next_completion_pos_bytes += buf1->len;
1870                 }
1871         }
1872
1873         /*
1874          * Setup the event to let the other thread know that it has work
1875          * pending.
1876          */
1877         EV_SET(&ke, (uintptr_t)&dev->peer_dev->work_queue, EVFILT_USER, 0,
1878                NOTE_TRIGGER, 0, NULL);
1879
1880         /*
1881          * Put this on our shadow queue so that we know what we've queued
1882          * to the other thread.
1883          */
1884         STAILQ_FOREACH_SAFE(buf1, &local_queue, work_links, buf2) {
1885                 if (buf1->buf_type != CAMDD_BUF_DATA) {
1886                         errx(1, "%s: should have a data buffer, not an "
1887                             "indirect buffer", __func__);
1888                 }
1889                 data = &buf1->buf_type_spec.data;
1890
1891                 /*
1892                  * We only need to send one EOF to the writer, and don't
1893                  * need to continue sending EOFs after that.
1894                  */
1895                 if (buf1->status == CAMDD_STATUS_EOF) {
1896                         if (dev->flags & CAMDD_DEV_FLAG_EOF_SENT) {
1897                                 STAILQ_REMOVE(&local_queue, buf1, camdd_buf,
1898                                     work_links);
1899                                 camdd_release_buf(buf1);
1900                                 retval = 1;
1901                                 continue;
1902                         }
1903                         dev->flags |= CAMDD_DEV_FLAG_EOF_SENT;
1904                 }
1905
1906
1907                 STAILQ_INSERT_TAIL(&dev->peer_work_queue, buf1, links);
1908                 peer_bytes_queued += (data->fill_len - data->resid);
1909                 dev->peer_bytes_queued += (data->fill_len - data->resid);
1910                 dev->num_peer_work_queue++;
1911         }
1912
1913         if (STAILQ_FIRST(&local_queue) == NULL)
1914                 goto bailout;
1915
1916         /*
1917          * Drop our mutex and pick up the other thread's mutex.  We need to
1918          * do this to avoid deadlocks.
1919          */
1920         pthread_mutex_unlock(&dev->mutex);
1921         pthread_mutex_lock(&dev->peer_dev->mutex);
1922
1923         if (dev->peer_dev->flags & CAMDD_DEV_FLAG_ACTIVE) {
1924                 /*
1925                  * Put the buffers on the other thread's incoming work queue.
1926                  */
1927                 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL;
1928                      buf1 = STAILQ_FIRST(&local_queue)) {
1929                         STAILQ_REMOVE_HEAD(&local_queue, work_links);
1930                         STAILQ_INSERT_TAIL(&dev->peer_dev->work_queue, buf1,
1931                                            work_links);
1932                 }
1933                 /*
1934                  * Send an event to the other thread's kqueue to let it know
1935                  * that there is something on the work queue.
1936                  */
1937                 retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL);
1938                 if (retval == -1)
1939                         warn("%s: unable to add peer work_queue kevent",
1940                              __func__);
1941                 else
1942                         retval = 0;
1943         } else
1944                 active = 0;
1945
1946         pthread_mutex_unlock(&dev->peer_dev->mutex);
1947         pthread_mutex_lock(&dev->mutex);
1948
1949         /*
1950          * If the other side isn't active, run through the queue and
1951          * release all of the buffers.
1952          */
1953         if (active == 0) {
1954                 for (buf1 = STAILQ_FIRST(&local_queue); buf1 != NULL;
1955                      buf1 = STAILQ_FIRST(&local_queue)) {
1956                         STAILQ_REMOVE_HEAD(&local_queue, work_links);
1957                         STAILQ_REMOVE(&dev->peer_work_queue, buf1, camdd_buf,
1958                                       links);
1959                         dev->num_peer_work_queue--;
1960                         camdd_release_buf(buf1);
1961                 }
1962                 dev->peer_bytes_queued -= peer_bytes_queued;
1963                 retval = 1;
1964         }
1965
1966 bailout:
1967         return (retval);
1968 }
1969
1970 /*
1971  * Return a buffer to the reader thread when we have completed writing it.
1972  */
1973 int
1974 camdd_complete_peer_buf(struct camdd_dev *dev, struct camdd_buf *peer_buf)
1975 {
1976         struct kevent ke;
1977         int retval = 0;
1978
1979         /*
1980          * Setup the event to let the other thread know that we have
1981          * completed a buffer.
1982          */
1983         EV_SET(&ke, (uintptr_t)&dev->peer_dev->peer_done_queue, EVFILT_USER, 0,
1984                NOTE_TRIGGER, 0, NULL);
1985
1986         /*
1987          * Drop our lock and acquire the other thread's lock before
1988          * manipulating
1989          */
1990         pthread_mutex_unlock(&dev->mutex);
1991         pthread_mutex_lock(&dev->peer_dev->mutex);
1992
1993         /*
1994          * Put the buffer on the reader thread's peer done queue now that
1995          * we have completed it.
1996          */
1997         STAILQ_INSERT_TAIL(&dev->peer_dev->peer_done_queue, peer_buf,
1998                            work_links);
1999         dev->peer_dev->num_peer_done_queue++;
2000
2001         /*
2002          * Send an event to the peer thread to let it know that we've added
2003          * something to its peer done queue.
2004          */
2005         retval = kevent(dev->peer_dev->kq, &ke, 1, NULL, 0, NULL);
2006         if (retval == -1)
2007                 warn("%s: unable to add peer_done_queue kevent", __func__);
2008         else
2009                 retval = 0;
2010
2011         /*
2012          * Drop the other thread's lock and reacquire ours.
2013          */
2014         pthread_mutex_unlock(&dev->peer_dev->mutex);
2015         pthread_mutex_lock(&dev->mutex);
2016
2017         return (retval);
2018 }
2019
2020 /*
2021  * Free a buffer that was written out by the writer thread and returned to
2022  * the reader thread.
2023  */
2024 void
2025 camdd_peer_done(struct camdd_buf *buf)
2026 {
2027         struct camdd_dev *dev;
2028         struct camdd_buf_data *data;
2029
2030         dev = buf->dev;
2031         if (buf->buf_type != CAMDD_BUF_DATA) {
2032                 errx(1, "%s: should have a data buffer, not an "
2033                     "indirect buffer", __func__);
2034         }
2035
2036         data = &buf->buf_type_spec.data;
2037
2038         STAILQ_REMOVE(&dev->peer_work_queue, buf, camdd_buf, links);
2039         dev->num_peer_work_queue--;
2040         dev->peer_bytes_queued -= (data->fill_len - data->resid);
2041
2042         if (buf->status == CAMDD_STATUS_EOF)
2043                 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF;
2044
2045         STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
2046 }
2047
2048 /*
2049  * Assumes caller holds the lock for this device.
2050  */
2051 void
2052 camdd_complete_buf(struct camdd_dev *dev, struct camdd_buf *buf,
2053                    int *error_count)
2054 {
2055         int retval = 0;
2056
2057         /*
2058          * If we're the reader, we need to send the completed I/O
2059          * to the writer.  If we're the writer, we need to just
2060          * free up resources, or let the reader know if we've
2061          * encountered an error.
2062          */
2063         if (dev->write_dev == 0) {
2064                 retval = camdd_queue_peer_buf(dev, buf);
2065                 if (retval != 0)
2066                         (*error_count)++;
2067         } else {
2068                 struct camdd_buf *tmp_buf, *next_buf;
2069
2070                 STAILQ_FOREACH_SAFE(tmp_buf, &buf->src_list, src_links,
2071                                     next_buf) {
2072                         struct camdd_buf *src_buf;
2073                         struct camdd_buf_indirect *indirect;
2074
2075                         STAILQ_REMOVE(&buf->src_list, tmp_buf,
2076                                       camdd_buf, src_links);
2077
2078                         tmp_buf->status = buf->status;
2079
2080                         if (tmp_buf->buf_type == CAMDD_BUF_DATA) {
2081                                 camdd_complete_peer_buf(dev, tmp_buf);
2082                                 continue;
2083                         }
2084
2085                         indirect = &tmp_buf->buf_type_spec.indirect;
2086                         src_buf = indirect->src_buf;
2087                         src_buf->refcount--;
2088                         /*
2089                          * XXX KDM we probably need to account for
2090                          * exactly how many bytes we were able to
2091                          * write.  Allocate the residual to the
2092                          * first N buffers?  Or just track the
2093                          * number of bytes written?  Right now the reader
2094                          * doesn't do anything with a residual.
2095                          */
2096                         src_buf->status = buf->status;
2097                         if (src_buf->refcount <= 0)
2098                                 camdd_complete_peer_buf(dev, src_buf);
2099                         STAILQ_INSERT_TAIL(&dev->free_indirect_queue,
2100                                            tmp_buf, links);
2101                 }
2102
2103                 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
2104         }
2105 }
2106
2107 /*
2108  * Fetch all completed commands from the pass(4) device.
2109  *
2110  * Returns the number of commands received, or -1 if any of the commands
2111  * completed with an error.  Returns 0 if no commands are available.
2112  */
2113 int
2114 camdd_pass_fetch(struct camdd_dev *dev)
2115 {
2116         struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass;
2117         union ccb ccb;
2118         int retval = 0, num_fetched = 0, error_count = 0;
2119
2120         pthread_mutex_unlock(&dev->mutex);
2121         /*
2122          * XXX KDM we don't distinguish between EFAULT and ENOENT.
2123          */
2124         while ((retval = ioctl(pass_dev->dev->fd, CAMIOGET, &ccb)) != -1) {
2125                 struct camdd_buf *buf;
2126                 struct camdd_buf_data *data;
2127                 cam_status ccb_status;
2128                 union ccb *buf_ccb;
2129
2130                 buf = ccb.ccb_h.ccb_buf;
2131                 data = &buf->buf_type_spec.data;
2132                 buf_ccb = &data->ccb;
2133
2134                 num_fetched++;
2135
2136                 /*
2137                  * Copy the CCB back out so we get status, sense data, etc.
2138                  */
2139                 bcopy(&ccb, buf_ccb, sizeof(ccb));
2140
2141                 pthread_mutex_lock(&dev->mutex);
2142
2143                 /*
2144                  * We're now done, so take this off the active queue.
2145                  */
2146                 STAILQ_REMOVE(&dev->active_queue, buf, camdd_buf, links);
2147                 dev->cur_active_io--;
2148
2149                 ccb_status = ccb.ccb_h.status & CAM_STATUS_MASK;
2150                 if (ccb_status != CAM_REQ_CMP) {
2151                         cam_error_print(pass_dev->dev, &ccb, CAM_ESF_ALL,
2152                                         CAM_EPF_ALL, stderr);
2153                 }
2154
2155                 data->resid = ccb.csio.resid;
2156                 dev->bytes_transferred += (ccb.csio.dxfer_len - ccb.csio.resid);
2157
2158                 if (buf->status == CAMDD_STATUS_NONE)
2159                         buf->status = camdd_ccb_status(&ccb);
2160                 if (buf->status == CAMDD_STATUS_ERROR)
2161                         error_count++;
2162                 else if (buf->status == CAMDD_STATUS_EOF) {
2163                         /*
2164                          * Once we queue this buffer to our partner thread,
2165                          * he will know that we've hit EOF.
2166                          */
2167                         dev->flags |= CAMDD_DEV_FLAG_EOF;
2168                 }
2169
2170                 camdd_complete_buf(dev, buf, &error_count);
2171
2172                 /*
2173                  * Unlock in preparation for the ioctl call.
2174                  */
2175                 pthread_mutex_unlock(&dev->mutex);
2176         }
2177
2178         pthread_mutex_lock(&dev->mutex);
2179
2180         if (error_count > 0)
2181                 return (-1);
2182         else
2183                 return (num_fetched);
2184 }
2185
2186 /*
2187  * Returns -1 for error, 0 for success/continue, and 1 for resource
2188  * shortage/stop processing.
2189  */
2190 int
2191 camdd_file_run(struct camdd_dev *dev)
2192 {
2193         struct camdd_dev_file *file_dev = &dev->dev_spec.file;
2194         struct camdd_buf_data *data;
2195         struct camdd_buf *buf;
2196         off_t io_offset;
2197         int retval = 0, write_dev = dev->write_dev;
2198         int error_count = 0, no_resources = 0, double_buf_needed = 0;
2199         uint32_t num_sectors = 0, db_len = 0;
2200
2201         buf = STAILQ_FIRST(&dev->run_queue);
2202         if (buf == NULL) {
2203                 no_resources = 1;
2204                 goto bailout;
2205         } else if ((dev->write_dev == 0)
2206                 && (dev->flags & (CAMDD_DEV_FLAG_EOF |
2207                                   CAMDD_DEV_FLAG_EOF_SENT))) {
2208                 STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links);
2209                 dev->num_run_queue--;
2210                 buf->status = CAMDD_STATUS_EOF;
2211                 error_count++;
2212                 goto bailout;
2213         }
2214
2215         /*
2216          * If we're writing, we need to go through the source buffer list
2217          * and create an S/G list.
2218          */
2219         if (write_dev != 0) {
2220                 retval = camdd_buf_sg_create(buf, /*iovec*/ 1,
2221                     dev->sector_size, &num_sectors, &double_buf_needed);
2222                 if (retval != 0) {
2223                         no_resources = 1;
2224                         goto bailout;
2225                 }
2226         }
2227
2228         STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links);
2229         dev->num_run_queue--;
2230
2231         data = &buf->buf_type_spec.data;
2232
2233         /*
2234          * pread(2) and pwrite(2) offsets are byte offsets.
2235          */
2236         io_offset = buf->lba * dev->sector_size;
2237
2238         /*
2239          * Unlock the mutex while we read or write.
2240          */
2241         pthread_mutex_unlock(&dev->mutex);
2242
2243         /*
2244          * Note that we don't need to double buffer if we're the reader
2245          * because in that case, we have allocated a single buffer of
2246          * sufficient size to do the read.  This copy is necessary on
2247          * writes because if one of the components of the S/G list is not
2248          * a sector size multiple, the kernel will reject the write.  This
2249          * is unfortunate but not surprising.  So this will make sure that
2250          * we're using a single buffer that is a multiple of the sector size.
2251          */
2252         if ((double_buf_needed != 0)
2253          && (data->sg_count > 1)
2254          && (write_dev != 0)) {
2255                 uint32_t cur_offset;
2256                 int i;
2257
2258                 if (file_dev->tmp_buf == NULL)
2259                         file_dev->tmp_buf = calloc(dev->blocksize, 1);
2260                 if (file_dev->tmp_buf == NULL) {
2261                         buf->status = CAMDD_STATUS_ERROR;
2262                         error_count++;
2263                         goto bailout;
2264                 }
2265                 for (i = 0, cur_offset = 0; i < data->sg_count; i++) {
2266                         bcopy(data->iovec[i].iov_base,
2267                             &file_dev->tmp_buf[cur_offset],
2268                             data->iovec[i].iov_len);
2269                         cur_offset += data->iovec[i].iov_len;
2270                 }
2271                 db_len = cur_offset;
2272         }
2273
2274         if (file_dev->file_flags & CAMDD_FF_CAN_SEEK) {
2275                 if (write_dev == 0) {
2276                         /*
2277                          * XXX KDM is there any way we would need a S/G
2278                          * list here?
2279                          */
2280                         retval = pread(file_dev->fd, data->buf,
2281                             buf->len, io_offset);
2282                 } else {
2283                         if (double_buf_needed != 0) {
2284                                 retval = pwrite(file_dev->fd, file_dev->tmp_buf,
2285                                     db_len, io_offset);
2286                         } else if (data->sg_count == 0) {
2287                                 retval = pwrite(file_dev->fd, data->buf,
2288                                     data->fill_len, io_offset);
2289                         } else {
2290                                 retval = pwritev(file_dev->fd, data->iovec,
2291                                     data->sg_count, io_offset);
2292                         }
2293                 }
2294         } else {
2295                 if (write_dev == 0) {
2296                         /*
2297                          * XXX KDM is there any way we would need a S/G
2298                          * list here?
2299                          */
2300                         retval = read(file_dev->fd, data->buf, buf->len);
2301                 } else {
2302                         if (double_buf_needed != 0) {
2303                                 retval = write(file_dev->fd, file_dev->tmp_buf,
2304                                     db_len);
2305                         } else if (data->sg_count == 0) {
2306                                 retval = write(file_dev->fd, data->buf,
2307                                     data->fill_len);
2308                         } else {
2309                                 retval = writev(file_dev->fd, data->iovec,
2310                                     data->sg_count);
2311                         }
2312                 }
2313         }
2314
2315         /* We're done, re-acquire the lock */
2316         pthread_mutex_lock(&dev->mutex);
2317
2318         if (retval >= (ssize_t)data->fill_len) {
2319                 /*
2320                  * If the bytes transferred is more than the request size,
2321                  * that indicates an overrun, which should only happen at
2322                  * the end of a transfer if we have to round up to a sector
2323                  * boundary.
2324                  */
2325                 if (buf->status == CAMDD_STATUS_NONE)
2326                         buf->status = CAMDD_STATUS_OK;
2327                 data->resid = 0;
2328                 dev->bytes_transferred += retval;
2329         } else if (retval == -1) {
2330                 warn("Error %s %s", (write_dev) ? "writing to" :
2331                     "reading from", file_dev->filename);
2332
2333                 buf->status = CAMDD_STATUS_ERROR;
2334                 data->resid = data->fill_len;
2335                 error_count++;
2336
2337                 if (dev->debug == 0)
2338                         goto bailout;
2339
2340                 if ((double_buf_needed != 0)
2341                  && (write_dev != 0)) {
2342                         fprintf(stderr, "%s: fd %d, DB buf %p, len %u lba %ju "
2343                             "offset %ju\n", __func__, file_dev->fd,
2344                             file_dev->tmp_buf, db_len, (uintmax_t)buf->lba,
2345                             (uintmax_t)io_offset);
2346                 } else if (data->sg_count == 0) {
2347                         fprintf(stderr, "%s: fd %d, buf %p, len %u, lba %ju "
2348                             "offset %ju\n", __func__, file_dev->fd, data->buf,
2349                             data->fill_len, (uintmax_t)buf->lba,
2350                             (uintmax_t)io_offset);
2351                 } else {
2352                         int i;
2353
2354                         fprintf(stderr, "%s: fd %d, len %u, lba %ju "
2355                             "offset %ju\n", __func__, file_dev->fd,
2356                             data->fill_len, (uintmax_t)buf->lba,
2357                             (uintmax_t)io_offset);
2358
2359                         for (i = 0; i < data->sg_count; i++) {
2360                                 fprintf(stderr, "index %d ptr %p len %zu\n",
2361                                     i, data->iovec[i].iov_base,
2362                                     data->iovec[i].iov_len);
2363                         }
2364                 }
2365         } else if (retval == 0) {
2366                 buf->status = CAMDD_STATUS_EOF;
2367                 if (dev->debug != 0)
2368                         printf("%s: got EOF from %s!\n", __func__,
2369                             file_dev->filename);
2370                 data->resid = data->fill_len;
2371                 error_count++;
2372         } else if (retval < (ssize_t)data->fill_len) {
2373                 if (buf->status == CAMDD_STATUS_NONE)
2374                         buf->status = CAMDD_STATUS_SHORT_IO;
2375                 data->resid = data->fill_len - retval;
2376                 dev->bytes_transferred += retval;
2377         }
2378
2379 bailout:
2380         if (buf != NULL) {
2381                 if (buf->status == CAMDD_STATUS_EOF) {
2382                         struct camdd_buf *buf2;
2383                         dev->flags |= CAMDD_DEV_FLAG_EOF;
2384                         STAILQ_FOREACH(buf2, &dev->run_queue, links)
2385                                 buf2->status = CAMDD_STATUS_EOF;
2386                 }
2387
2388                 camdd_complete_buf(dev, buf, &error_count);
2389         }
2390
2391         if (error_count != 0)
2392                 return (-1);
2393         else if (no_resources != 0)
2394                 return (1);
2395         else
2396                 return (0);
2397 }
2398
2399 /*
2400  * Execute one command from the run queue.  Returns 0 for success, 1 for
2401  * stop processing, and -1 for error.
2402  */
2403 int
2404 camdd_pass_run(struct camdd_dev *dev)
2405 {
2406         struct camdd_buf *buf = NULL;
2407         struct camdd_dev_pass *pass_dev = &dev->dev_spec.pass;
2408         struct camdd_buf_data *data;
2409         uint32_t num_blocks, sectors_used = 0;
2410         union ccb *ccb;
2411         int retval = 0, is_write = dev->write_dev;
2412         int double_buf_needed = 0;
2413
2414         buf = STAILQ_FIRST(&dev->run_queue);
2415         if (buf == NULL) {
2416                 retval = 1;
2417                 goto bailout;
2418         }
2419
2420         /*
2421          * If we're writing, we need to go through the source buffer list
2422          * and create an S/G list.
2423          */
2424         if (is_write != 0) {
2425                 retval = camdd_buf_sg_create(buf, /*iovec*/ 0,dev->sector_size,
2426                     &sectors_used, &double_buf_needed);
2427                 if (retval != 0) {
2428                         retval = -1;
2429                         goto bailout;
2430                 }
2431         }
2432
2433         STAILQ_REMOVE(&dev->run_queue, buf, camdd_buf, links);
2434         dev->num_run_queue--;
2435
2436         data = &buf->buf_type_spec.data;
2437
2438         ccb = &data->ccb;
2439         bzero(&(&ccb->ccb_h)[1],
2440               sizeof(struct ccb_scsiio) - sizeof(struct ccb_hdr));
2441
2442         /*
2443          * In almost every case the number of blocks should be the device
2444          * block size.  The exception may be at the end of an I/O stream
2445          * for a partial block or at the end of a device.
2446          */
2447         if (is_write != 0)
2448                 num_blocks = sectors_used;
2449         else
2450                 num_blocks = data->fill_len / pass_dev->block_len;
2451
2452         scsi_read_write(&ccb->csio,
2453                         /*retries*/ dev->retry_count,
2454                         /*cbfcnp*/ NULL,
2455                         /*tag_action*/ MSG_SIMPLE_Q_TAG,
2456                         /*readop*/ (dev->write_dev == 0) ? SCSI_RW_READ :
2457                                    SCSI_RW_WRITE,
2458                         /*byte2*/ 0,
2459                         /*minimum_cmd_size*/ dev->min_cmd_size,
2460                         /*lba*/ buf->lba,
2461                         /*block_count*/ num_blocks,
2462                         /*data_ptr*/ (data->sg_count != 0) ?
2463                                      (uint8_t *)data->segs : data->buf,
2464                         /*dxfer_len*/ (num_blocks * pass_dev->block_len),
2465                         /*sense_len*/ SSD_FULL_SIZE,
2466                         /*timeout*/ dev->io_timeout);
2467
2468         /* Disable freezing the device queue */
2469         ccb->ccb_h.flags |= CAM_DEV_QFRZDIS;
2470
2471         if (dev->retry_count != 0)
2472                 ccb->ccb_h.flags |= CAM_PASS_ERR_RECOVER;
2473
2474         if (data->sg_count != 0) {
2475                 ccb->csio.sglist_cnt = data->sg_count;
2476                 ccb->ccb_h.flags |= CAM_DATA_SG;
2477         }
2478
2479         /*
2480          * Store a pointer to the buffer in the CCB.  The kernel will
2481          * restore this when we get it back, and we'll use it to identify
2482          * the buffer this CCB came from.
2483          */
2484         ccb->ccb_h.ccb_buf = buf;
2485
2486         /*
2487          * Unlock our mutex in preparation for issuing the ioctl.
2488          */
2489         pthread_mutex_unlock(&dev->mutex);
2490         /*
2491          * Queue the CCB to the pass(4) driver.
2492          */
2493         if (ioctl(pass_dev->dev->fd, CAMIOQUEUE, ccb) == -1) {
2494                 pthread_mutex_lock(&dev->mutex);
2495
2496                 warn("%s: error sending CAMIOQUEUE ioctl to %s%u", __func__,
2497                      pass_dev->dev->device_name, pass_dev->dev->dev_unit_num);
2498                 warn("%s: CCB address is %p", __func__, ccb);
2499                 retval = -1;
2500
2501                 STAILQ_INSERT_TAIL(&dev->free_queue, buf, links);
2502         } else {
2503                 pthread_mutex_lock(&dev->mutex);
2504
2505                 dev->cur_active_io++;
2506                 STAILQ_INSERT_TAIL(&dev->active_queue, buf, links);
2507         }
2508
2509 bailout:
2510         return (retval);
2511 }
2512
2513 int
2514 camdd_get_next_lba_len(struct camdd_dev *dev, uint64_t *lba, ssize_t *len)
2515 {
2516         struct camdd_dev_pass *pass_dev;
2517         uint32_t num_blocks;
2518         int retval = 0;
2519
2520         pass_dev = &dev->dev_spec.pass;
2521
2522         *lba = dev->next_io_pos_bytes / dev->sector_size;
2523         *len = dev->blocksize;
2524         num_blocks = *len / dev->sector_size;
2525
2526         /*
2527          * If max_sector is 0, then we have no set limit.  This can happen
2528          * if we're writing to a file in a filesystem, or reading from
2529          * something like /dev/zero.
2530          */
2531         if ((dev->max_sector != 0)
2532          || (dev->sector_io_limit != 0)) {
2533                 uint64_t max_sector;
2534
2535                 if ((dev->max_sector != 0)
2536                  && (dev->sector_io_limit != 0))
2537                         max_sector = min(dev->sector_io_limit, dev->max_sector);
2538                 else if (dev->max_sector != 0)
2539                         max_sector = dev->max_sector;
2540                 else
2541                         max_sector = dev->sector_io_limit;
2542
2543
2544                 /*
2545                  * Check to see whether we're starting off past the end of
2546                  * the device.  If so, we need to just send an EOF
2547                  * notification to the writer.
2548                  */
2549                 if (*lba > max_sector) {
2550                         *len = 0;
2551                         retval = 1;
2552                 } else if (((*lba + num_blocks) > max_sector + 1)
2553                         || ((*lba + num_blocks) < *lba)) {
2554                         /*
2555                          * If we get here (but pass the first check), we
2556                          * can trim the request length down to go to the
2557                          * end of the device.
2558                          */
2559                         num_blocks = (max_sector + 1) - *lba;
2560                         *len = num_blocks * dev->sector_size;
2561                         retval = 1;
2562                 }
2563         }
2564
2565         dev->next_io_pos_bytes += *len;
2566
2567         return (retval);
2568 }
2569
2570 /*
2571  * Returns 0 for success, 1 for EOF detected, and -1 for failure.
2572  */
2573 int
2574 camdd_queue(struct camdd_dev *dev, struct camdd_buf *read_buf)
2575 {
2576         struct camdd_buf *buf = NULL;
2577         struct camdd_buf_data *data;
2578         struct camdd_dev_pass *pass_dev;
2579         size_t new_len;
2580         struct camdd_buf_data *rb_data;
2581         int is_write = dev->write_dev;
2582         int eof_flush_needed = 0;
2583         int retval = 0;
2584         int error;
2585
2586         pass_dev = &dev->dev_spec.pass;
2587
2588         /*
2589          * If we've gotten EOF or our partner has, we should not continue
2590          * queueing I/O.  If we're a writer, though, we should continue
2591          * to write any buffers that don't have EOF status.
2592          */
2593         if ((dev->flags & CAMDD_DEV_FLAG_EOF)
2594          || ((dev->flags & CAMDD_DEV_FLAG_PEER_EOF)
2595           && (is_write == 0))) {
2596                 /*
2597                  * Tell the worker thread that we have seen EOF.
2598                  */
2599                 retval = 1;
2600
2601                 /*
2602                  * If we're the writer, send the buffer back with EOF status.
2603                  */
2604                 if (is_write) {
2605                         read_buf->status = CAMDD_STATUS_EOF;
2606
2607                         error = camdd_complete_peer_buf(dev, read_buf);
2608                 }
2609                 goto bailout;
2610         }
2611
2612         if (is_write == 0) {
2613                 buf = camdd_get_buf(dev, CAMDD_BUF_DATA);
2614                 if (buf == NULL) {
2615                         retval = -1;
2616                         goto bailout;
2617                 }
2618                 data = &buf->buf_type_spec.data;
2619
2620                 retval = camdd_get_next_lba_len(dev, &buf->lba, &buf->len);
2621                 if (retval != 0) {
2622                         buf->status = CAMDD_STATUS_EOF;
2623
2624                         if ((buf->len == 0)
2625                          && ((dev->flags & (CAMDD_DEV_FLAG_EOF_SENT |
2626                              CAMDD_DEV_FLAG_EOF_QUEUED)) != 0)) {
2627                                 camdd_release_buf(buf);
2628                                 goto bailout;
2629                         }
2630                         dev->flags |= CAMDD_DEV_FLAG_EOF_QUEUED;
2631                 }
2632
2633                 data->fill_len = buf->len;
2634                 data->src_start_offset = buf->lba * dev->sector_size;
2635
2636                 /*
2637                  * Put this on the run queue.
2638                  */
2639                 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
2640                 dev->num_run_queue++;
2641
2642                 /* We're done. */
2643                 goto bailout;
2644         }
2645
2646         /*
2647          * Check for new EOF status from the reader.
2648          */
2649         if ((read_buf->status == CAMDD_STATUS_EOF)
2650          || (read_buf->status == CAMDD_STATUS_ERROR)) {
2651                 dev->flags |= CAMDD_DEV_FLAG_PEER_EOF;
2652                 if ((STAILQ_FIRST(&dev->pending_queue) == NULL)
2653                  && (read_buf->len == 0)) {
2654                         camdd_complete_peer_buf(dev, read_buf);
2655                         retval = 1;
2656                         goto bailout;
2657                 } else
2658                         eof_flush_needed = 1;
2659         }
2660
2661         /*
2662          * See if we have a buffer we're composing with pieces from our
2663          * partner thread.
2664          */
2665         buf = STAILQ_FIRST(&dev->pending_queue);
2666         if (buf == NULL) {
2667                 uint64_t lba;
2668                 ssize_t len;
2669
2670                 retval = camdd_get_next_lba_len(dev, &lba, &len);
2671                 if (retval != 0) {
2672                         read_buf->status = CAMDD_STATUS_EOF;
2673
2674                         if (len == 0) {
2675                                 dev->flags |= CAMDD_DEV_FLAG_EOF;
2676                                 error = camdd_complete_peer_buf(dev, read_buf);
2677                                 goto bailout;
2678                         }
2679                 }
2680
2681                 /*
2682                  * If we don't have a pending buffer, we need to grab a new
2683                  * one from the free list or allocate another one.
2684                  */
2685                 buf = camdd_get_buf(dev, CAMDD_BUF_DATA);
2686                 if (buf == NULL) {
2687                         retval = 1;
2688                         goto bailout;
2689                 }
2690
2691                 buf->lba = lba;
2692                 buf->len = len;
2693
2694                 STAILQ_INSERT_TAIL(&dev->pending_queue, buf, links);
2695                 dev->num_pending_queue++;
2696         }
2697
2698         data = &buf->buf_type_spec.data;
2699
2700         rb_data = &read_buf->buf_type_spec.data;
2701
2702         if ((rb_data->src_start_offset != dev->next_peer_pos_bytes)
2703          && (dev->debug != 0)) {
2704                 printf("%s: WARNING: reader offset %#jx != expected offset "
2705                     "%#jx\n", __func__, (uintmax_t)rb_data->src_start_offset,
2706                     (uintmax_t)dev->next_peer_pos_bytes);
2707         }
2708         dev->next_peer_pos_bytes = rb_data->src_start_offset +
2709             (rb_data->fill_len - rb_data->resid);
2710
2711         new_len = (rb_data->fill_len - rb_data->resid) + data->fill_len;
2712         if (new_len < buf->len) {
2713                 /*
2714                  * There are three cases here:
2715                  * 1. We need more data to fill up a block, so we put
2716                  *    this I/O on the queue and wait for more I/O.
2717                  * 2. We have a pending buffer in the queue that is
2718                  *    smaller than our blocksize, but we got an EOF.  So we
2719                  *    need to go ahead and flush the write out.
2720                  * 3. We got an error.
2721                  */
2722
2723                 /*
2724                  * Increment our fill length.
2725                  */
2726                 data->fill_len += (rb_data->fill_len - rb_data->resid);
2727
2728                 /*
2729                  * Add the new read buffer to the list for writing.
2730                  */
2731                 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links);
2732
2733                 /* Increment the count */
2734                 buf->src_count++;
2735
2736                 if (eof_flush_needed == 0) {
2737                         /*
2738                          * We need to exit, because we don't have enough
2739                          * data yet.
2740                          */
2741                         goto bailout;
2742                 } else {
2743                         /*
2744                          * Take the buffer off of the pending queue.
2745                          */
2746                         STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf,
2747                                       links);
2748                         dev->num_pending_queue--;
2749
2750                         /*
2751                          * If we need an EOF flush, but there is no data
2752                          * to flush, go ahead and return this buffer.
2753                          */
2754                         if (data->fill_len == 0) {
2755                                 camdd_complete_buf(dev, buf, /*error_count*/0);
2756                                 retval = 1;
2757                                 goto bailout;
2758                         }
2759
2760                         /*
2761                          * Put this on the next queue for execution.
2762                          */
2763                         STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
2764                         dev->num_run_queue++;
2765                 }
2766         } else if (new_len == buf->len) {
2767                 /*
2768                  * We have enough data to completey fill one block,
2769                  * so we're ready to issue the I/O.
2770                  */
2771
2772                 /*
2773                  * Take the buffer off of the pending queue.
2774                  */
2775                 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf, links);
2776                 dev->num_pending_queue--;
2777
2778                 /*
2779                  * Add the new read buffer to the list for writing.
2780                  */
2781                 STAILQ_INSERT_TAIL(&buf->src_list, read_buf, src_links);
2782
2783                 /* Increment the count */
2784                 buf->src_count++;
2785
2786                 /*
2787                  * Increment our fill length.
2788                  */
2789                 data->fill_len += (rb_data->fill_len - rb_data->resid);
2790
2791                 /*
2792                  * Put this on the next queue for execution.
2793                  */
2794                 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
2795                 dev->num_run_queue++;
2796         } else {
2797                 struct camdd_buf *idb;
2798                 struct camdd_buf_indirect *indirect;
2799                 uint32_t len_to_go, cur_offset;
2800
2801
2802                 idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT);
2803                 if (idb == NULL) {
2804                         retval = 1;
2805                         goto bailout;
2806                 }
2807                 indirect = &idb->buf_type_spec.indirect;
2808                 indirect->src_buf = read_buf;
2809                 read_buf->refcount++;
2810                 indirect->offset = 0;
2811                 indirect->start_ptr = rb_data->buf;
2812                 /*
2813                  * We've already established that there is more
2814                  * data in read_buf than we have room for in our
2815                  * current write request.  So this particular chunk
2816                  * of the request should just be the remainder
2817                  * needed to fill up a block.
2818                  */
2819                 indirect->len = buf->len - (data->fill_len - data->resid);
2820
2821                 camdd_buf_add_child(buf, idb);
2822
2823                 /*
2824                  * This buffer is ready to execute, so we can take
2825                  * it off the pending queue and put it on the run
2826                  * queue.
2827                  */
2828                 STAILQ_REMOVE(&dev->pending_queue, buf, camdd_buf,
2829                               links);
2830                 dev->num_pending_queue--;
2831                 STAILQ_INSERT_TAIL(&dev->run_queue, buf, links);
2832                 dev->num_run_queue++;
2833
2834                 cur_offset = indirect->offset + indirect->len;
2835
2836                 /*
2837                  * The resulting I/O would be too large to fit in
2838                  * one block.  We need to split this I/O into
2839                  * multiple pieces.  Allocate as many buffers as needed.
2840                  */
2841                 for (len_to_go = rb_data->fill_len - rb_data->resid -
2842                      indirect->len; len_to_go > 0;) {
2843                         struct camdd_buf *new_buf;
2844                         struct camdd_buf_data *new_data;
2845                         uint64_t lba;
2846                         ssize_t len;
2847
2848                         retval = camdd_get_next_lba_len(dev, &lba, &len);
2849                         if ((retval != 0)
2850                          && (len == 0)) {
2851                                 /*
2852                                  * The device has already been marked
2853                                  * as EOF, and there is no space left.
2854                                  */
2855                                 goto bailout;
2856                         }
2857
2858                         new_buf = camdd_get_buf(dev, CAMDD_BUF_DATA);
2859                         if (new_buf == NULL) {
2860                                 retval = 1;
2861                                 goto bailout;
2862                         }
2863
2864                         new_buf->lba = lba;
2865                         new_buf->len = len;
2866
2867                         idb = camdd_get_buf(dev, CAMDD_BUF_INDIRECT);
2868                         if (idb == NULL) {
2869                                 retval = 1;
2870                                 goto bailout;
2871                         }
2872
2873                         indirect = &idb->buf_type_spec.indirect;
2874
2875                         indirect->src_buf = read_buf;
2876                         read_buf->refcount++;
2877                         indirect->offset = cur_offset;
2878                         indirect->start_ptr = rb_data->buf + cur_offset;
2879                         indirect->len = min(len_to_go, new_buf->len);
2880 #if 0
2881                         if (((indirect->len % dev->sector_size) != 0)
2882                          || ((indirect->offset % dev->sector_size) != 0)) {
2883                                 warnx("offset %ju len %ju not aligned with "
2884                                     "sector size %u", indirect->offset,
2885                                     (uintmax_t)indirect->len, dev->sector_size);
2886                         }
2887 #endif
2888                         cur_offset += indirect->len;
2889                         len_to_go -= indirect->len;
2890
2891                         camdd_buf_add_child(new_buf, idb);
2892
2893                         new_data = &new_buf->buf_type_spec.data;
2894
2895                         if ((new_data->fill_len == new_buf->len)
2896                          || (eof_flush_needed != 0)) {
2897                                 STAILQ_INSERT_TAIL(&dev->run_queue,
2898                                                    new_buf, links);
2899                                 dev->num_run_queue++;
2900                         } else if (new_data->fill_len < buf->len) {
2901                                 STAILQ_INSERT_TAIL(&dev->pending_queue,
2902                                                 new_buf, links);
2903                                 dev->num_pending_queue++;
2904                         } else {
2905                                 warnx("%s: too much data in new "
2906                                       "buffer!", __func__);
2907                                 retval = 1;
2908                                 goto bailout;
2909                         }
2910                 }
2911         }
2912
2913 bailout:
2914         return (retval);
2915 }
2916
2917 void
2918 camdd_get_depth(struct camdd_dev *dev, uint32_t *our_depth,
2919                 uint32_t *peer_depth, uint32_t *our_bytes, uint32_t *peer_bytes)
2920 {
2921         *our_depth = dev->cur_active_io + dev->num_run_queue;
2922         if (dev->num_peer_work_queue >
2923             dev->num_peer_done_queue)
2924                 *peer_depth = dev->num_peer_work_queue -
2925                               dev->num_peer_done_queue;
2926         else
2927                 *peer_depth = 0;
2928         *our_bytes = *our_depth * dev->blocksize;
2929         *peer_bytes = dev->peer_bytes_queued;
2930 }
2931
2932 void
2933 camdd_sig_handler(int sig)
2934 {
2935         if (sig == SIGINFO)
2936                 need_status = 1;
2937         else {
2938                 need_exit = 1;
2939                 error_exit = 1;
2940         }
2941
2942         sem_post(&camdd_sem);
2943 }
2944
2945 void
2946 camdd_print_status(struct camdd_dev *camdd_dev, struct camdd_dev *other_dev,
2947                    struct timespec *start_time)
2948 {
2949         struct timespec done_time;
2950         uint64_t total_ns;
2951         long double mb_sec, total_sec;
2952         int error = 0;
2953
2954         error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &done_time);
2955         if (error != 0) {
2956                 warn("Unable to get done time");
2957                 return;
2958         }
2959
2960         timespecsub(&done_time, start_time);
2961
2962         total_ns = done_time.tv_nsec + (done_time.tv_sec * 1000000000);
2963         total_sec = total_ns;
2964         total_sec /= 1000000000;
2965
2966         fprintf(stderr, "%ju bytes %s %s\n%ju bytes %s %s\n"
2967                 "%.4Lf seconds elapsed\n",
2968                 (uintmax_t)camdd_dev->bytes_transferred,
2969                 (camdd_dev->write_dev == 0) ?  "read from" : "written to",
2970                 camdd_dev->device_name,
2971                 (uintmax_t)other_dev->bytes_transferred,
2972                 (other_dev->write_dev == 0) ? "read from" : "written to",
2973                 other_dev->device_name, total_sec);
2974
2975         mb_sec = min(other_dev->bytes_transferred,camdd_dev->bytes_transferred);
2976         mb_sec /= 1024 * 1024;
2977         mb_sec *= 1000000000;
2978         mb_sec /= total_ns;
2979         fprintf(stderr, "%.2Lf MB/sec\n", mb_sec);
2980 }
2981
2982 int
2983 camdd_rw(struct camdd_io_opts *io_opts, int num_io_opts, uint64_t max_io,
2984          int retry_count, int timeout)
2985 {
2986         char *device = NULL;
2987         struct cam_device *new_cam_dev = NULL;
2988         struct camdd_dev *devs[2];
2989         struct timespec start_time;
2990         pthread_t threads[2];
2991         int unit = 0;
2992         int error = 0;
2993         int i;
2994
2995         if (num_io_opts != 2) {
2996                 warnx("Must have one input and one output path");
2997                 error = 1;
2998                 goto bailout;
2999         }
3000
3001         bzero(devs, sizeof(devs));
3002
3003         for (i = 0; i < num_io_opts; i++) {
3004                 switch (io_opts[i].dev_type) {
3005                 case CAMDD_DEV_PASS: {
3006                         camdd_argmask new_arglist = CAMDD_ARG_NONE;
3007                         int bus = 0, target = 0, lun = 0;
3008                         char name[30];
3009                         int rv;
3010
3011                         if (isdigit(io_opts[i].dev_name[0])) {
3012                                 /* device specified as bus:target[:lun] */
3013                                 rv = parse_btl(io_opts[i].dev_name, &bus,
3014                                     &target, &lun, &new_arglist);
3015                                 if (rv < 2) {
3016                                         warnx("numeric device specification "
3017                                              "must be either bus:target, or "
3018                                              "bus:target:lun");
3019                                         error = 1;
3020                                         goto bailout;
3021                                 }
3022                                 /* default to 0 if lun was not specified */
3023                                 if ((new_arglist & CAMDD_ARG_LUN) == 0) {
3024                                         lun = 0;
3025                                         new_arglist |= CAMDD_ARG_LUN;
3026                                 }
3027                         } else {
3028                                 if (cam_get_device(io_opts[i].dev_name, name,
3029                                                    sizeof name, &unit) == -1) {
3030                                         warnx("%s", cam_errbuf);
3031                                         error = 1;
3032                                         goto bailout;
3033                                 }
3034                                 device = strdup(name);
3035                                 new_arglist |= CAMDD_ARG_DEVICE |CAMDD_ARG_UNIT;
3036                         }
3037
3038                         if (new_arglist & (CAMDD_ARG_BUS | CAMDD_ARG_TARGET))
3039                                 new_cam_dev = cam_open_btl(bus, target, lun,
3040                                     O_RDWR, NULL);
3041                         else
3042                                 new_cam_dev = cam_open_spec_device(device, unit,
3043                                     O_RDWR, NULL);
3044                         if (new_cam_dev == NULL) {
3045                                 warnx("%s", cam_errbuf);
3046                                 error = 1;
3047                                 goto bailout;
3048                         }
3049
3050                         devs[i] = camdd_probe_pass(new_cam_dev,
3051                             /*io_opts*/ &io_opts[i],
3052                             CAMDD_ARG_ERR_RECOVER,
3053                             /*probe_retry_count*/ 3,
3054                             /*probe_timeout*/ 5000,
3055                             /*io_retry_count*/ retry_count,
3056                             /*io_timeout*/ timeout);
3057                         if (devs[i] == NULL) {
3058                                 warn("Unable to probe device %s%u",
3059                                      new_cam_dev->device_name,
3060                                      new_cam_dev->dev_unit_num);
3061                                 error = 1;
3062                                 goto bailout;
3063                         }
3064                         break;
3065                 }
3066                 case CAMDD_DEV_FILE: {
3067                         int fd = -1;
3068
3069                         if (io_opts[i].dev_name[0] == '-') {
3070                                 if (io_opts[i].write_dev != 0)
3071                                         fd = STDOUT_FILENO;
3072                                 else
3073                                         fd = STDIN_FILENO;
3074                         } else {
3075                                 if (io_opts[i].write_dev != 0) {
3076                                         fd = open(io_opts[i].dev_name,
3077                                             O_RDWR | O_CREAT, S_IWUSR |S_IRUSR);
3078                                 } else {
3079                                         fd = open(io_opts[i].dev_name,
3080                                             O_RDONLY);
3081                                 }
3082                         }
3083                         if (fd == -1) {
3084                                 warn("error opening file %s",
3085                                     io_opts[i].dev_name);
3086                                 error = 1;
3087                                 goto bailout;
3088                         }
3089
3090                         devs[i] = camdd_probe_file(fd, &io_opts[i],
3091                             retry_count, timeout);
3092                         if (devs[i] == NULL) {
3093                                 error = 1;
3094                                 goto bailout;
3095                         }
3096
3097                         break;
3098                 }
3099                 default:
3100                         warnx("Unknown device type %d (%s)",
3101                             io_opts[i].dev_type, io_opts[i].dev_name);
3102                         error = 1;
3103                         goto bailout;
3104                         break; /*NOTREACHED */
3105                 }
3106
3107                 devs[i]->write_dev = io_opts[i].write_dev;
3108
3109                 devs[i]->start_offset_bytes = io_opts[i].offset;
3110
3111                 if (max_io != 0) {
3112                         devs[i]->sector_io_limit =
3113                             (devs[i]->start_offset_bytes /
3114                             devs[i]->sector_size) +
3115                             (max_io / devs[i]->sector_size) - 1;
3116                         devs[i]->sector_io_limit =
3117                             (devs[i]->start_offset_bytes /
3118                             devs[i]->sector_size) +
3119                             (max_io / devs[i]->sector_size) - 1;
3120                 }
3121
3122                 devs[i]->next_io_pos_bytes = devs[i]->start_offset_bytes;
3123                 devs[i]->next_completion_pos_bytes =devs[i]->start_offset_bytes;
3124         }
3125
3126         devs[0]->peer_dev = devs[1];
3127         devs[1]->peer_dev = devs[0];
3128         devs[0]->next_peer_pos_bytes = devs[0]->peer_dev->next_io_pos_bytes;
3129         devs[1]->next_peer_pos_bytes = devs[1]->peer_dev->next_io_pos_bytes;
3130
3131         sem_init(&camdd_sem, /*pshared*/ 0, 0);
3132
3133         signal(SIGINFO, camdd_sig_handler);
3134         signal(SIGINT, camdd_sig_handler);
3135
3136         error = clock_gettime(CLOCK_MONOTONIC_PRECISE, &start_time);
3137         if (error != 0) {
3138                 warn("Unable to get start time");
3139                 goto bailout;
3140         }
3141
3142         for (i = 0; i < num_io_opts; i++) {
3143                 error = pthread_create(&threads[i], NULL, camdd_worker,
3144                                        (void *)devs[i]);
3145                 if (error != 0) {
3146                         warnc(error, "pthread_create() failed");
3147                         goto bailout;
3148                 }
3149         }
3150
3151         for (;;) {
3152                 if ((sem_wait(&camdd_sem) == -1)
3153                  || (need_exit != 0)) {
3154                         struct kevent ke;
3155
3156                         for (i = 0; i < num_io_opts; i++) {
3157                                 EV_SET(&ke, (uintptr_t)&devs[i]->work_queue,
3158                                     EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
3159
3160                                 devs[i]->flags |= CAMDD_DEV_FLAG_EOF;
3161
3162                                 error = kevent(devs[i]->kq, &ke, 1, NULL, 0,
3163                                                 NULL);
3164                                 if (error == -1)
3165                                         warn("%s: unable to wake up thread",
3166                                             __func__);
3167                                 error = 0;
3168                         }
3169                         break;
3170                 } else if (need_status != 0) {
3171                         camdd_print_status(devs[0], devs[1], &start_time);
3172                         need_status = 0;
3173                 }
3174         }
3175         for (i = 0; i < num_io_opts; i++) {
3176                 pthread_join(threads[i], NULL);
3177         }
3178
3179         camdd_print_status(devs[0], devs[1], &start_time);
3180
3181 bailout:
3182
3183         for (i = 0; i < num_io_opts; i++)
3184                 camdd_free_dev(devs[i]);
3185
3186         return (error + error_exit);
3187 }
3188
3189 void
3190 usage(void)
3191 {
3192         fprintf(stderr,
3193 "usage:  camdd <-i|-o pass=pass0,bs=1M,offset=1M,depth=4>\n"
3194 "              <-i|-o file=/tmp/file,bs=512K,offset=1M>\n"
3195 "              <-i|-o file=/dev/da0,bs=512K,offset=1M>\n"
3196 "              <-i|-o file=/dev/nsa0,bs=512K>\n"
3197 "              [-C retry_count][-E][-m max_io_amt][-t timeout_secs][-v][-h]\n"
3198 "Option description\n"
3199 "-i <arg=val>  Specify input device/file and parameters\n"
3200 "-o <arg=val>  Specify output device/file and parameters\n"
3201 "Input and Output parameters\n"
3202 "pass=name     Specify a pass(4) device like pass0 or /dev/pass0\n"
3203 "file=name     Specify a file or device, /tmp/foo, /dev/da0, /dev/null\n"
3204 "              or - for stdin/stdout\n"
3205 "bs=blocksize  Specify blocksize in bytes, or using K, M, G, etc. suffix\n"
3206 "offset=len    Specify starting offset in bytes or using K, M, G suffix\n"
3207 "              NOTE: offset cannot be specified on tapes, pipes, stdin/out\n"
3208 "depth=N       Specify a numeric queue depth.  This only applies to pass(4)\n"
3209 "mcs=N         Specify a minimum cmd size for pass(4) read/write commands\n"
3210 "Optional arguments\n"
3211 "-C retry_cnt  Specify a retry count for pass(4) devices\n"
3212 "-E            Enable CAM error recovery for pass(4) devices\n"
3213 "-m max_io     Specify the maximum amount to be transferred in bytes or\n"
3214 "              using K, G, M, etc. suffixes\n"
3215 "-t timeout    Specify the I/O timeout to use with pass(4) devices\n"
3216 "-v            Enable verbose error recovery\n"
3217 "-h            Print this message\n");
3218 }
3219
3220
3221 int
3222 camdd_parse_io_opts(char *args, int is_write, struct camdd_io_opts *io_opts)
3223 {
3224         char *tmpstr, *tmpstr2;
3225         char *orig_tmpstr = NULL;
3226         int retval = 0;
3227
3228         io_opts->write_dev = is_write;
3229
3230         tmpstr = strdup(args);
3231         if (tmpstr == NULL) {
3232                 warn("strdup failed");
3233                 retval = 1;
3234                 goto bailout;
3235         }
3236         orig_tmpstr = tmpstr;
3237         while ((tmpstr2 = strsep(&tmpstr, ",")) != NULL) {
3238                 char *name, *value;
3239
3240                 /*
3241                  * If the user creates an empty parameter by putting in two
3242                  * commas, skip over it and look for the next field.
3243                  */
3244                 if (*tmpstr2 == '\0')
3245                         continue;
3246
3247                 name = strsep(&tmpstr2, "=");
3248                 if (*name == '\0') {
3249                         warnx("Got empty I/O parameter name");
3250                         retval = 1;
3251                         goto bailout;
3252                 }
3253                 value = strsep(&tmpstr2, "=");
3254                 if ((value == NULL)
3255                  || (*value == '\0')) {
3256                         warnx("Empty I/O parameter value for %s", name);
3257                         retval = 1;
3258                         goto bailout;
3259                 }
3260                 if (strncasecmp(name, "file", 4) == 0) {
3261                         io_opts->dev_type = CAMDD_DEV_FILE;
3262                         io_opts->dev_name = strdup(value);
3263                         if (io_opts->dev_name == NULL) {
3264                                 warn("Error allocating memory");
3265                                 retval = 1;
3266                                 goto bailout;
3267                         }
3268                 } else if (strncasecmp(name, "pass", 4) == 0) {
3269                         io_opts->dev_type = CAMDD_DEV_PASS;
3270                         io_opts->dev_name = strdup(value);
3271                         if (io_opts->dev_name == NULL) {
3272                                 warn("Error allocating memory");
3273                                 retval = 1;
3274                                 goto bailout;
3275                         }
3276                 } else if ((strncasecmp(name, "bs", 2) == 0)
3277                         || (strncasecmp(name, "blocksize", 9) == 0)) {
3278                         retval = expand_number(value, &io_opts->blocksize);
3279                         if (retval == -1) {
3280                                 warn("expand_number(3) failed on %s=%s", name,
3281                                     value);
3282                                 retval = 1;
3283                                 goto bailout;
3284                         }
3285                 } else if (strncasecmp(name, "depth", 5) == 0) {
3286                         char *endptr;
3287
3288                         io_opts->queue_depth = strtoull(value, &endptr, 0);
3289                         if (*endptr != '\0') {
3290                                 warnx("invalid queue depth %s", value);
3291                                 retval = 1;
3292                                 goto bailout;
3293                         }
3294                 } else if (strncasecmp(name, "mcs", 3) == 0) {
3295                         char *endptr;
3296
3297                         io_opts->min_cmd_size = strtol(value, &endptr, 0);
3298                         if ((*endptr != '\0')
3299                          || ((io_opts->min_cmd_size > 16)
3300                           || (io_opts->min_cmd_size < 0))) {
3301                                 warnx("invalid minimum cmd size %s", value);
3302                                 retval = 1;
3303                                 goto bailout;
3304                         }
3305                 } else if (strncasecmp(name, "offset", 6) == 0) {
3306                         retval = expand_number(value, &io_opts->offset);
3307                         if (retval == -1) {
3308                                 warn("expand_number(3) failed on %s=%s", name,
3309                                     value);
3310                                 retval = 1;
3311                                 goto bailout;
3312                         }
3313                 } else if (strncasecmp(name, "debug", 5) == 0) {
3314                         char *endptr;
3315
3316                         io_opts->debug = strtoull(value, &endptr, 0);
3317                         if (*endptr != '\0') {
3318                                 warnx("invalid debug level %s", value);
3319                                 retval = 1;
3320                                 goto bailout;
3321                         }
3322                 } else {
3323                         warnx("Unrecognized parameter %s=%s", name, value);
3324                 }
3325         }
3326 bailout:
3327         free(orig_tmpstr);
3328
3329         return (retval);
3330 }
3331
3332 int
3333 main(int argc, char **argv)
3334 {
3335         int c;
3336         camdd_argmask arglist = CAMDD_ARG_NONE;
3337         int timeout = 0, retry_count = 1;
3338         int error = 0;
3339         uint64_t max_io = 0;
3340         struct camdd_io_opts *opt_list = NULL;
3341
3342         if (argc == 1) {
3343                 usage();
3344                 exit(1);
3345         }
3346
3347         opt_list = calloc(2, sizeof(struct camdd_io_opts));
3348         if (opt_list == NULL) {
3349                 warn("Unable to allocate option list");
3350                 error = 1;
3351                 goto bailout;
3352         }
3353
3354         while ((c = getopt(argc, argv, "C:Ehi:m:o:t:v")) != -1){
3355                 switch (c) {
3356                 case 'C':
3357                         retry_count = strtol(optarg, NULL, 0);
3358                         if (retry_count < 0)
3359                                 errx(1, "retry count %d is < 0",
3360                                      retry_count);
3361                         arglist |= CAMDD_ARG_RETRIES;
3362                         break;
3363                 case 'E':
3364                         arglist |= CAMDD_ARG_ERR_RECOVER;
3365                         break;
3366                 case 'i':
3367                 case 'o':
3368                         if (((c == 'i')
3369                           && (opt_list[0].dev_type != CAMDD_DEV_NONE))
3370                          || ((c == 'o')
3371                           && (opt_list[1].dev_type != CAMDD_DEV_NONE))) {
3372                                 errx(1, "Only one input and output path "
3373                                     "allowed");
3374                         }
3375                         error = camdd_parse_io_opts(optarg, (c == 'o') ? 1 : 0,
3376                             (c == 'o') ? &opt_list[1] : &opt_list[0]);
3377                         if (error != 0)
3378                                 goto bailout;
3379                         break;
3380                 case 'm':
3381                         error = expand_number(optarg, &max_io);
3382                         if (error == -1) {
3383                                 warn("invalid maximum I/O amount %s", optarg);
3384                                 error = 1;
3385                                 goto bailout;
3386                         }
3387                         break;
3388                 case 't':
3389                         timeout = strtol(optarg, NULL, 0);
3390                         if (timeout < 0)
3391                                 errx(1, "invalid timeout %d", timeout);
3392                         /* Convert the timeout from seconds to ms */
3393                         timeout *= 1000;
3394                         arglist |= CAMDD_ARG_TIMEOUT;
3395                         break;
3396                 case 'v':
3397                         arglist |= CAMDD_ARG_VERBOSE;
3398                         break;
3399                 case 'h':
3400                 default:
3401                         usage();
3402                         exit(1);
3403                         break; /*NOTREACHED*/
3404                 }
3405         }
3406
3407         if ((opt_list[0].dev_type == CAMDD_DEV_NONE)
3408          || (opt_list[1].dev_type == CAMDD_DEV_NONE))
3409                 errx(1, "Must specify both -i and -o");
3410
3411         /*
3412          * Set the timeout if the user hasn't specified one.
3413          */
3414         if (timeout == 0)
3415                 timeout = CAMDD_PASS_RW_TIMEOUT;
3416
3417         error = camdd_rw(opt_list, 2, max_io, retry_count, timeout);
3418
3419 bailout:
3420         free(opt_list);
3421
3422         exit(error);
3423 }