sys/cam/scsi/scsi_pass.c

   1 /*-
   2  * Copyright (c) 1997, 1998, 2000 Justin T. Gibbs.
   3  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions, and the following disclaimer,
  11  *    without modification, immediately at the beginning of the file.
  12  * 2. The name of the author may not be used to endorse or promote products
  13  *    derived from this software without specific prior written permission.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  19  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_kdtrace.h"
  32
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/kernel.h>
  36 #include <sys/conf.h>
  37 #include <sys/types.h>
  38 #include <sys/bio.h>
  39 #include <sys/bus.h>
  40 #include <sys/devicestat.h>
  41 #include <sys/errno.h>
  42 #include <sys/fcntl.h>
  43 #include <sys/malloc.h>
  44 #include <sys/proc.h>
  45 #include <sys/poll.h>
  46 #include <sys/selinfo.h>
  47 #include <sys/sdt.h>
  48 #include <sys/taskqueue.h>
  49 #include <vm/uma.h>
  50 #include <vm/vm.h>
  51 #include <vm/vm_extern.h>
  52
  53 #include <machine/bus.h>
  54
  55 #include <cam/cam.h>
  56 #include <cam/cam_ccb.h>
  57 #include <cam/cam_periph.h>
  58 #include <cam/cam_queue.h>
  59 #include <cam/cam_xpt.h>
  60 #include <cam/cam_xpt_periph.h>
  61 #include <cam/cam_debug.h>
  62 #include <cam/cam_compat.h>
  63 #include <cam/cam_xpt_periph.h>
  64
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_pass.h>
  67
  68 typedef enum {
  69         PASS_FLAG_OPEN                  = 0x01,
  70         PASS_FLAG_LOCKED                = 0x02,
  71         PASS_FLAG_INVALID               = 0x04,
  72         PASS_FLAG_INITIAL_PHYSPATH      = 0x08,
  73         PASS_FLAG_ZONE_INPROG           = 0x10,
  74         PASS_FLAG_ZONE_VALID            = 0x20,
  75         PASS_FLAG_UNMAPPED_CAPABLE      = 0x40,
  76         PASS_FLAG_ABANDONED_REF_SET     = 0x80
  77 } pass_flags;
  78
  79 typedef enum {
  80         PASS_STATE_NORMAL
  81 } pass_state;
  82
  83 typedef enum {
  84         PASS_CCB_BUFFER_IO,
  85         PASS_CCB_QUEUED_IO
  86 } pass_ccb_types;
  87
  88 #define ccb_type        ppriv_field0
  89 #define ccb_ioreq       ppriv_ptr1
  90
  91 /*
  92  * The maximum number of memory segments we preallocate.
  93  */
  94 #define PASS_MAX_SEGS   16
  95
  96 typedef enum {
  97         PASS_IO_NONE            = 0x00,
  98         PASS_IO_USER_SEG_MALLOC = 0x01,
  99         PASS_IO_KERN_SEG_MALLOC = 0x02,
 100         PASS_IO_ABANDONED       = 0x04
 101 } pass_io_flags;
 102
 103 struct pass_io_req {
 104         union ccb                        ccb;
 105         union ccb                       *alloced_ccb;
 106         union ccb                       *user_ccb_ptr;
 107         camq_entry                       user_periph_links;
 108         ccb_ppriv_area                   user_periph_priv;
 109         struct cam_periph_map_info       mapinfo;
 110         pass_io_flags                    flags;
 111         ccb_flags                        data_flags;
 112         int                              num_user_segs;
 113         bus_dma_segment_t                user_segs[PASS_MAX_SEGS];
 114         int                              num_kern_segs;
 115         bus_dma_segment_t                kern_segs[PASS_MAX_SEGS];
 116         bus_dma_segment_t               *user_segptr;
 117         bus_dma_segment_t               *kern_segptr;
 118         int                              num_bufs;
 119         uint32_t                         dirs[CAM_PERIPH_MAXMAPS];
 120         uint32_t                         lengths[CAM_PERIPH_MAXMAPS];
 121         uint8_t                         *user_bufs[CAM_PERIPH_MAXMAPS];
 122         uint8_t                         *kern_bufs[CAM_PERIPH_MAXMAPS];
 123         struct bintime                   start_time;
 124         TAILQ_ENTRY(pass_io_req)         links;
 125 };
 126
 127 struct pass_softc {
 128         pass_state                state;
 129         pass_flags                flags;
 130         u_int8_t                  pd_type;
 131         union ccb                 saved_ccb;
 132         int                       open_count;
 133         u_int                     maxio;
 134         struct devstat           *device_stats;
 135         struct cdev              *dev;
 136         struct cdev              *alias_dev;
 137         struct task               add_physpath_task;
 138         struct task               shutdown_kqueue_task;
 139         struct selinfo            read_select;
 140         TAILQ_HEAD(, pass_io_req) incoming_queue;
 141         TAILQ_HEAD(, pass_io_req) active_queue;
 142         TAILQ_HEAD(, pass_io_req) abandoned_queue;
 143         TAILQ_HEAD(, pass_io_req) done_queue;
 144         struct cam_periph        *periph;
 145         char                      zone_name[12];
 146         char                      io_zone_name[12];
 147         uma_zone_t                pass_zone;
 148         uma_zone_t                pass_io_zone;
 149         size_t                    io_zone_size;
 150 };
 151
 152 static  d_open_t        passopen;
 153 static  d_close_t       passclose;
 154 static  d_ioctl_t       passioctl;
 155 static  d_ioctl_t       passdoioctl;
 156 static  d_poll_t        passpoll;
 157 static  d_kqfilter_t    passkqfilter;
 158 static  void            passreadfiltdetach(struct knote *kn);
 159 static  int             passreadfilt(struct knote *kn, long hint);
 160
 161 static  periph_init_t   passinit;
 162 static  periph_ctor_t   passregister;
 163 static  periph_oninv_t  passoninvalidate;
 164 static  periph_dtor_t   passcleanup;
 165 static  periph_start_t  passstart;
 166 static  void            pass_shutdown_kqueue(void *context, int pending);
 167 static  void            pass_add_physpath(void *context, int pending);
 168 static  void            passasync(void *callback_arg, u_int32_t code,
 169                                   struct cam_path *path, void *arg);
 170 static  void            passdone(struct cam_periph *periph,
 171                                  union ccb *done_ccb);
 172 static  int             passcreatezone(struct cam_periph *periph);
 173 static  void            passiocleanup(struct pass_softc *softc,
 174                                       struct pass_io_req *io_req);
 175 static  int             passcopysglist(struct cam_periph *periph,
 176                                        struct pass_io_req *io_req,
 177                                        ccb_flags direction);
 178 static  int             passmemsetup(struct cam_periph *periph,
 179                                      struct pass_io_req *io_req);
 180 static  int             passmemdone(struct cam_periph *periph,
 181                                     struct pass_io_req *io_req);
 182 static  int             passerror(union ccb *ccb, u_int32_t cam_flags,
 183                                   u_int32_t sense_flags);
 184 static  int             passsendccb(struct cam_periph *periph, union ccb *ccb,
 185                                     union ccb *inccb);
 186
 187 static struct periph_driver passdriver =
 188 {
 189         passinit, "pass",
 190         TAILQ_HEAD_INITIALIZER(passdriver.units), /* generation */ 0
 191 };
 192
 193 PERIPHDRIVER_DECLARE(pass, passdriver);
 194
 195 static struct cdevsw pass_cdevsw = {
 196         .d_version =    D_VERSION,
 197         .d_flags =      D_TRACKCLOSE,
 198         .d_open =       passopen,
 199         .d_close =      passclose,
 200         .d_ioctl =      passioctl,
 201         .d_poll =       passpoll,
 202         .d_kqfilter =   passkqfilter,
 203         .d_name =       "pass",
 204 };
 205
 206 static struct filterops passread_filtops = {
 207         .f_isfd =       1,
 208         .f_detach =     passreadfiltdetach,
 209         .f_event =      passreadfilt
 210 };
 211
 212 static MALLOC_DEFINE(M_SCSIPASS, "scsi_pass", "scsi passthrough buffers");
 213
 214 static void
 215 passinit(void)
 216 {
 217         cam_status status;
 218
 219         /*
 220          * Install a global async callback.  This callback will
 221          * receive async callbacks like "new device found".
 222          */
 223         status = xpt_register_async(AC_FOUND_DEVICE, passasync, NULL, NULL);
 224
 225         if (status != CAM_REQ_CMP) {
 226                 printf("pass: Failed to attach master async callback "
 227                        "due to status 0x%x!\n", status);
 228         }
 229
 230 }
 231
 232 static void
 233 passrejectios(struct cam_periph *periph)
 234 {
 235         struct pass_io_req *io_req, *io_req2;
 236         struct pass_softc *softc;
 237
 238         softc = (struct pass_softc *)periph->softc;
 239
 240         /*
 241          * The user can no longer get status for I/O on the done queue, so
 242          * clean up all outstanding I/O on the done queue.
 243          */
 244         TAILQ_FOREACH_SAFE(io_req, &softc->done_queue, links, io_req2) {
 245                 TAILQ_REMOVE(&softc->done_queue, io_req, links);
 246                 passiocleanup(softc, io_req);
 247                 uma_zfree(softc->pass_zone, io_req);
 248         }
 249
 250         /*
 251          * The underlying device is gone, so we can't issue these I/Os.
 252          * The devfs node has been shut down, so we can't return status to
 253          * the user.  Free any I/O left on the incoming queue.
 254          */
 255         TAILQ_FOREACH_SAFE(io_req, &softc->incoming_queue, links, io_req2) {
 256                 TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
 257                 passiocleanup(softc, io_req);
 258                 uma_zfree(softc->pass_zone, io_req);
 259         }
 260
 261         /*
 262          * Normally we would put I/Os on the abandoned queue and acquire a
 263          * reference when we saw the final close.  But, the device went
 264          * away and devfs may have moved everything off to deadfs by the
 265          * time the I/O done callback is called; as a result, we won't see
 266          * any more closes.  So, if we have any active I/Os, we need to put
 267          * them on the abandoned queue.  When the abandoned queue is empty,
 268          * we'll release the remaining reference (see below) to the peripheral.
 269          */
 270         TAILQ_FOREACH_SAFE(io_req, &softc->active_queue, links, io_req2) {
 271                 TAILQ_REMOVE(&softc->active_queue, io_req, links);
 272                 io_req->flags |= PASS_IO_ABANDONED;
 273                 TAILQ_INSERT_TAIL(&softc->abandoned_queue, io_req, links);
 274         }
 275
 276         /*
 277          * If we put any I/O on the abandoned queue, acquire a reference.
 278          */
 279         if ((!TAILQ_EMPTY(&softc->abandoned_queue))
 280          && ((softc->flags & PASS_FLAG_ABANDONED_REF_SET) == 0)) {
 281                 cam_periph_doacquire(periph);
 282                 softc->flags |= PASS_FLAG_ABANDONED_REF_SET;
 283         }
 284 }
 285
 286 static void
 287 passdevgonecb(void *arg)
 288 {
 289         struct cam_periph *periph;
 290         struct mtx *mtx;
 291         struct pass_softc *softc;
 292         int i;
 293
 294         periph = (struct cam_periph *)arg;
 295         mtx = cam_periph_mtx(periph);
 296         mtx_lock(mtx);
 297
 298         softc = (struct pass_softc *)periph->softc;
 299         KASSERT(softc->open_count >= 0, ("Negative open count %d",
 300                 softc->open_count));
 301
 302         /*
 303          * When we get this callback, we will get no more close calls from
 304          * devfs.  So if we have any dangling opens, we need to release the
 305          * reference held for that particular context.
 306          */
 307         for (i = 0; i < softc->open_count; i++)
 308                 cam_periph_release_locked(periph);
 309
 310         softc->open_count = 0;
 311
 312         /*
 313          * Release the reference held for the device node, it is gone now.
 314          * Accordingly, inform all queued I/Os of their fate.
 315          */
 316         cam_periph_release_locked(periph);
 317         passrejectios(periph);
 318
 319         /*
 320          * We reference the SIM lock directly here, instead of using
 321          * cam_periph_unlock().  The reason is that the final call to
 322          * cam_periph_release_locked() above could result in the periph
 323          * getting freed.  If that is the case, dereferencing the periph
 324          * with a cam_periph_unlock() call would cause a page fault.
 325          */
 326         mtx_unlock(mtx);
 327
 328         /*
 329          * We have to remove our kqueue context from a thread because it
 330          * may sleep.  It would be nice if we could get a callback from
 331          * kqueue when it is done cleaning up resources.
 332          */
 333         taskqueue_enqueue(taskqueue_thread, &softc->shutdown_kqueue_task);
 334 }
 335
 336 static void
 337 passoninvalidate(struct cam_periph *periph)
 338 {
 339         struct pass_softc *softc;
 340
 341         softc = (struct pass_softc *)periph->softc;
 342
 343         /*
 344          * De-register any async callbacks.
 345          */
 346         xpt_register_async(0, passasync, periph, periph->path);
 347
 348         softc->flags |= PASS_FLAG_INVALID;
 349
 350         /*
 351          * Tell devfs this device has gone away, and ask for a callback
 352          * when it has cleaned up its state.
 353          */
 354         destroy_dev_sched_cb(softc->dev, passdevgonecb, periph);
 355 }
 356
 357 static void
 358 passcleanup(struct cam_periph *periph)
 359 {
 360         struct pass_softc *softc;
 361
 362         softc = (struct pass_softc *)periph->softc;
 363
 364         cam_periph_assert(periph, MA_OWNED);
 365         KASSERT(TAILQ_EMPTY(&softc->active_queue),
 366                 ("%s called when there are commands on the active queue!\n",
 367                 __func__));
 368         KASSERT(TAILQ_EMPTY(&softc->abandoned_queue),
 369                 ("%s called when there are commands on the abandoned queue!\n",
 370                 __func__));
 371         KASSERT(TAILQ_EMPTY(&softc->incoming_queue),
 372                 ("%s called when there are commands on the incoming queue!\n",
 373                 __func__));
 374         KASSERT(TAILQ_EMPTY(&softc->done_queue),
 375                 ("%s called when there are commands on the done queue!\n",
 376                 __func__));
 377
 378         devstat_remove_entry(softc->device_stats);
 379
 380         cam_periph_unlock(periph);
 381
 382         /*
 383          * We call taskqueue_drain() for the physpath task to make sure it
 384          * is complete.  We drop the lock because this can potentially
 385          * sleep.  XXX KDM that is bad.  Need a way to get a callback when
 386          * a taskqueue is drained.
 387          *
 388          * Note that we don't drain the kqueue shutdown task queue.  This
 389          * is because we hold a reference on the periph for kqueue, and
 390          * release that reference from the kqueue shutdown task queue.  So
 391          * we cannot come into this routine unless we've released that
 392          * reference.  Also, because that could be the last reference, we
 393          * could be called from the cam_periph_release() call in
 394          * pass_shutdown_kqueue().  In that case, the taskqueue_drain()
 395          * would deadlock.  It would be preferable if we had a way to
 396          * get a callback when a taskqueue is done.
 397          */
 398         taskqueue_drain(taskqueue_thread, &softc->add_physpath_task);
 399
 400         cam_periph_lock(periph);
 401
 402         free(softc, M_DEVBUF);
 403 }
 404
 405 static void
 406 pass_shutdown_kqueue(void *context, int pending)
 407 {
 408         struct cam_periph *periph;
 409         struct pass_softc *softc;
 410
 411         periph = context;
 412         softc = periph->softc;
 413
 414         knlist_clear(&softc->read_select.si_note, /*is_locked*/ 0);
 415         knlist_destroy(&softc->read_select.si_note);
 416
 417         /*
 418          * Release the reference we held for kqueue.
 419          */
 420         cam_periph_release(periph);
 421 }
 422
 423 static void
 424 pass_add_physpath(void *context, int pending)
 425 {
 426         struct cam_periph *periph;
 427         struct pass_softc *softc;
 428         struct mtx *mtx;
 429         char *physpath;
 430
 431         /*
 432          * If we have one, create a devfs alias for our
 433          * physical path.
 434          */
 435         periph = context;
 436         softc = periph->softc;
 437         physpath = malloc(MAXPATHLEN, M_DEVBUF, M_WAITOK);
 438         mtx = cam_periph_mtx(periph);
 439         mtx_lock(mtx);
 440
 441         if (periph->flags & CAM_PERIPH_INVALID)
 442                 goto out;
 443
 444         if (xpt_getattr(physpath, MAXPATHLEN,
 445                         "GEOM::physpath", periph->path) == 0
 446          && strlen(physpath) != 0) {
 447
 448                 mtx_unlock(mtx);
 449                 make_dev_physpath_alias(MAKEDEV_WAITOK, &softc->alias_dev,
 450                                         softc->dev, softc->alias_dev, physpath);
 451                 mtx_lock(mtx);
 452         }
 453
 454 out:
 455         /*
 456          * Now that we've made our alias, we no longer have to have a
 457          * reference to the device.
 458          */
 459         if ((softc->flags & PASS_FLAG_INITIAL_PHYSPATH) == 0)
 460                 softc->flags |= PASS_FLAG_INITIAL_PHYSPATH;
 461
 462         /*
 463          * We always acquire a reference to the periph before queueing this
 464          * task queue function, so it won't go away before we run.
 465          */
 466         while (pending-- > 0)
 467                 cam_periph_release_locked(periph);
 468         mtx_unlock(mtx);
 469
 470         free(physpath, M_DEVBUF);
 471 }
 472
 473 static void
 474 passasync(void *callback_arg, u_int32_t code,
 475           struct cam_path *path, void *arg)
 476 {
 477         struct cam_periph *periph;
 478
 479         periph = (struct cam_periph *)callback_arg;
 480
 481         switch (code) {
 482         case AC_FOUND_DEVICE:
 483         {
 484                 struct ccb_getdev *cgd;
 485                 cam_status status;
 486
 487                 cgd = (struct ccb_getdev *)arg;
 488                 if (cgd == NULL)
 489                         break;
 490
 491                 /*
 492                  * Allocate a peripheral instance for
 493                  * this device and start the probe
 494                  * process.
 495                  */
 496                 status = cam_periph_alloc(passregister, passoninvalidate,
 497                                           passcleanup, passstart, "pass",
 498                                           CAM_PERIPH_BIO, path,
 499                                           passasync, AC_FOUND_DEVICE, cgd);
 500
 501                 if (status != CAM_REQ_CMP
 502                  && status != CAM_REQ_INPROG) {
 503                         const struct cam_status_entry *entry;
 504
 505                         entry = cam_fetch_status_entry(status);
 506
 507                         printf("passasync: Unable to attach new device "
 508                                "due to status %#x: %s\n", status, entry ?
 509                                entry->status_text : "Unknown");
 510                 }
 511
 512                 break;
 513         }
 514         case AC_ADVINFO_CHANGED:
 515         {
 516                 uintptr_t buftype;
 517
 518                 buftype = (uintptr_t)arg;
 519                 if (buftype == CDAI_TYPE_PHYS_PATH) {
 520                         struct pass_softc *softc;
 521                         cam_status status;
 522
 523                         softc = (struct pass_softc *)periph->softc;
 524                         /*
 525                          * Acquire a reference to the periph before we
 526                          * start the taskqueue, so that we don't run into
 527                          * a situation where the periph goes away before
 528                          * the task queue has a chance to run.
 529                          */
 530                         status = cam_periph_acquire(periph);
 531                         if (status != CAM_REQ_CMP)
 532                                 break;
 533
 534                         taskqueue_enqueue(taskqueue_thread,
 535                                           &softc->add_physpath_task);
 536                 }
 537                 break;
 538         }
 539         default:
 540                 cam_periph_async(periph, code, path, arg);
 541                 break;
 542         }
 543 }
 544
 545 static cam_status
 546 passregister(struct cam_periph *periph, void *arg)
 547 {
 548         struct pass_softc *softc;
 549         struct ccb_getdev *cgd;
 550         struct ccb_pathinq cpi;
 551         struct make_dev_args args;
 552         int error, no_tags;
 553
 554         cgd = (struct ccb_getdev *)arg;
 555         if (cgd == NULL) {
 556                 printf("%s: no getdev CCB, can't register device\n", __func__);
 557                 return(CAM_REQ_CMP_ERR);
 558         }
 559
 560         softc = (struct pass_softc *)malloc(sizeof(*softc),
 561                                             M_DEVBUF, M_NOWAIT);
 562
 563         if (softc == NULL) {
 564                 printf("%s: Unable to probe new device. "
 565                        "Unable to allocate softc\n", __func__);
 566                 return(CAM_REQ_CMP_ERR);
 567         }
 568
 569         bzero(softc, sizeof(*softc));
 570         softc->state = PASS_STATE_NORMAL;
 571         if (cgd->protocol == PROTO_SCSI || cgd->protocol == PROTO_ATAPI)
 572                 softc->pd_type = SID_TYPE(&cgd->inq_data);
 573         else if (cgd->protocol == PROTO_SATAPM)
 574                 softc->pd_type = T_ENCLOSURE;
 575         else
 576                 softc->pd_type = T_DIRECT;
 577
 578         periph->softc = softc;
 579         softc->periph = periph;
 580         TAILQ_INIT(&softc->incoming_queue);
 581         TAILQ_INIT(&softc->active_queue);
 582         TAILQ_INIT(&softc->abandoned_queue);
 583         TAILQ_INIT(&softc->done_queue);
 584         snprintf(softc->zone_name, sizeof(softc->zone_name), "%s%d",
 585                  periph->periph_name, periph->unit_number);
 586         snprintf(softc->io_zone_name, sizeof(softc->io_zone_name), "%s%dIO",
 587                  periph->periph_name, periph->unit_number);
 588         softc->io_zone_size = MAXPHYS;
 589         knlist_init_mtx(&softc->read_select.si_note, cam_periph_mtx(periph));
 590
 591         bzero(&cpi, sizeof(cpi));
 592         xpt_setup_ccb(&cpi.ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 593         cpi.ccb_h.func_code = XPT_PATH_INQ;
 594         xpt_action((union ccb *)&cpi);
 595
 596         if (cpi.maxio == 0)
 597                 softc->maxio = DFLTPHYS;        /* traditional default */
 598         else if (cpi.maxio > MAXPHYS)
 599                 softc->maxio = MAXPHYS;         /* for safety */
 600         else
 601                 softc->maxio = cpi.maxio;       /* real value */
 602
 603         if (cpi.hba_misc & PIM_UNMAPPED)
 604                 softc->flags |= PASS_FLAG_UNMAPPED_CAPABLE;
 605
 606         /*
 607          * We pass in 0 for a blocksize, since we don't
 608          * know what the blocksize of this device is, if
 609          * it even has a blocksize.
 610          */
 611         cam_periph_unlock(periph);
 612         no_tags = (cgd->inq_data.flags & SID_CmdQue) == 0;
 613         softc->device_stats = devstat_new_entry("pass",
 614                           periph->unit_number, 0,
 615                           DEVSTAT_NO_BLOCKSIZE
 616                           | (no_tags ? DEVSTAT_NO_ORDERED_TAGS : 0),
 617                           softc->pd_type |
 618                           XPORT_DEVSTAT_TYPE(cpi.transport) |
 619                           DEVSTAT_TYPE_PASS,
 620                           DEVSTAT_PRIORITY_PASS);
 621
 622         /*
 623          * Initialize the taskqueue handler for shutting down kqueue.
 624          */
 625         TASK_INIT(&softc->shutdown_kqueue_task, /*priority*/ 0,
 626                   pass_shutdown_kqueue, periph);
 627
 628         /*
 629          * Acquire a reference to the periph that we can release once we've
 630          * cleaned up the kqueue.
 631          */
 632         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 633                 xpt_print(periph->path, "%s: lost periph during "
 634                           "registration!\n", __func__);
 635                 cam_periph_lock(periph);
 636                 return (CAM_REQ_CMP_ERR);
 637         }
 638
 639         /*
 640          * Acquire a reference to the periph before we create the devfs
 641          * instance for it.  We'll release this reference once the devfs
 642          * instance has been freed.
 643          */
 644         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 645                 xpt_print(periph->path, "%s: lost periph during "
 646                           "registration!\n", __func__);
 647                 cam_periph_lock(periph);
 648                 return (CAM_REQ_CMP_ERR);
 649         }
 650
 651         /* Register the device */
 652         make_dev_args_init(&args);
 653         args.mda_devsw = &pass_cdevsw;
 654         args.mda_unit = periph->unit_number;
 655         args.mda_uid = UID_ROOT;
 656         args.mda_gid = GID_OPERATOR;
 657         args.mda_mode = 0600;
 658         args.mda_si_drv1 = periph;
 659         error = make_dev_s(&args, &softc->dev, "%s%d", periph->periph_name,
 660             periph->unit_number);
 661         if (error != 0) {
 662                 cam_periph_lock(periph);
 663                 cam_periph_release_locked(periph);
 664                 return (CAM_REQ_CMP_ERR);
 665         }
 666
 667         /*
 668          * Hold a reference to the periph before we create the physical
 669          * path alias so it can't go away.
 670          */
 671         if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 672                 xpt_print(periph->path, "%s: lost periph during "
 673                           "registration!\n", __func__);
 674                 cam_periph_lock(periph);
 675                 return (CAM_REQ_CMP_ERR);
 676         }
 677
 678         cam_periph_lock(periph);
 679
 680         TASK_INIT(&softc->add_physpath_task, /*priority*/0,
 681                   pass_add_physpath, periph);
 682
 683         /*
 684          * See if physical path information is already available.
 685          */
 686         taskqueue_enqueue(taskqueue_thread, &softc->add_physpath_task);
 687
 688         /*
 689          * Add an async callback so that we get notified if
 690          * this device goes away or its physical path
 691          * (stored in the advanced info data of the EDT) has
 692          * changed.
 693          */
 694         xpt_register_async(AC_LOST_DEVICE | AC_ADVINFO_CHANGED,
 695                            passasync, periph, periph->path);
 696
 697         if (bootverbose)
 698                 xpt_announce_periph(periph, NULL);
 699
 700         return(CAM_REQ_CMP);
 701 }
 702
 703 static int
 704 passopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 705 {
 706         struct cam_periph *periph;
 707         struct pass_softc *softc;
 708         int error;
 709
 710         periph = (struct cam_periph *)dev->si_drv1;
 711         if (cam_periph_acquire(periph) != CAM_REQ_CMP)
 712                 return (ENXIO);
 713
 714         cam_periph_lock(periph);
 715
 716         softc = (struct pass_softc *)periph->softc;
 717
 718         if (softc->flags & PASS_FLAG_INVALID) {
 719                 cam_periph_release_locked(periph);
 720                 cam_periph_unlock(periph);
 721                 return(ENXIO);
 722         }
 723
 724         /*
 725          * Don't allow access when we're running at a high securelevel.
 726          */
 727         error = securelevel_gt(td->td_ucred, 1);
 728         if (error) {
 729                 cam_periph_release_locked(periph);
 730                 cam_periph_unlock(periph);
 731                 return(error);
 732         }
 733
 734         /*
 735          * Only allow read-write access.
 736          */
 737         if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0)) {
 738                 cam_periph_release_locked(periph);
 739                 cam_periph_unlock(periph);
 740                 return(EPERM);
 741         }
 742
 743         /*
 744          * We don't allow nonblocking access.
 745          */
 746         if ((flags & O_NONBLOCK) != 0) {
 747                 xpt_print(periph->path, "can't do nonblocking access\n");
 748                 cam_periph_release_locked(periph);
 749                 cam_periph_unlock(periph);
 750                 return(EINVAL);
 751         }
 752
 753         softc->open_count++;
 754
 755         cam_periph_unlock(periph);
 756
 757         return (error);
 758 }
 759
 760 static int
 761 passclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 762 {
 763         struct  cam_periph *periph;
 764         struct  pass_softc *softc;
 765         struct mtx *mtx;
 766
 767         periph = (struct cam_periph *)dev->si_drv1;
 768         mtx = cam_periph_mtx(periph);
 769         mtx_lock(mtx);
 770
 771         softc = periph->softc;
 772         softc->open_count--;
 773
 774         if (softc->open_count == 0) {
 775                 struct pass_io_req *io_req, *io_req2;
 776                 int need_unlock;
 777
 778                 need_unlock = 0;
 779
 780                 TAILQ_FOREACH_SAFE(io_req, &softc->done_queue, links, io_req2) {
 781                         TAILQ_REMOVE(&softc->done_queue, io_req, links);
 782                         passiocleanup(softc, io_req);
 783                         uma_zfree(softc->pass_zone, io_req);
 784                 }
 785
 786                 TAILQ_FOREACH_SAFE(io_req, &softc->incoming_queue, links,
 787                                    io_req2) {
 788                         TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
 789                         passiocleanup(softc, io_req);
 790                         uma_zfree(softc->pass_zone, io_req);
 791                 }
 792
 793                 /*
 794                  * If there are any active I/Os, we need to forcibly acquire a
 795                  * reference to the peripheral so that we don't go away
 796                  * before they complete.  We'll release the reference when
 797                  * the abandoned queue is empty.
 798                  */
 799                 io_req = TAILQ_FIRST(&softc->active_queue);
 800                 if ((io_req != NULL)
 801                  && (softc->flags & PASS_FLAG_ABANDONED_REF_SET) == 0) {
 802                         cam_periph_doacquire(periph);
 803                         softc->flags |= PASS_FLAG_ABANDONED_REF_SET;
 804                 }
 805
 806                 /*
 807                  * Since the I/O in the active queue is not under our
 808                  * control, just set a flag so that we can clean it up when
 809                  * it completes and put it on the abandoned queue.  This
 810                  * will prevent our sending spurious completions in the
 811                  * event that the device is opened again before these I/Os
 812                  * complete.
 813                  */
 814                 TAILQ_FOREACH_SAFE(io_req, &softc->active_queue, links,
 815                                    io_req2) {
 816                         TAILQ_REMOVE(&softc->active_queue, io_req, links);
 817                         io_req->flags |= PASS_IO_ABANDONED;
 818                         TAILQ_INSERT_TAIL(&softc->abandoned_queue, io_req,
 819                                           links);
 820                 }
 821         }
 822
 823         cam_periph_release_locked(periph);
 824
 825         /*
 826          * We reference the lock directly here, instead of using
 827          * cam_periph_unlock().  The reason is that the call to
 828          * cam_periph_release_locked() above could result in the periph
 829          * getting freed.  If that is the case, dereferencing the periph
 830          * with a cam_periph_unlock() call would cause a page fault.
 831          *
 832          * cam_periph_release() avoids this problem using the same method,
 833          * but we're manually acquiring and dropping the lock here to
 834          * protect the open count and avoid another lock acquisition and
 835          * release.
 836          */
 837         mtx_unlock(mtx);
 838
 839         return (0);
 840 }
 841
 842
 843 static void
 844 passstart(struct cam_periph *periph, union ccb *start_ccb)
 845 {
 846         struct pass_softc *softc;
 847
 848         softc = (struct pass_softc *)periph->softc;
 849
 850         switch (softc->state) {
 851         case PASS_STATE_NORMAL: {
 852                 struct pass_io_req *io_req;
 853
 854                 /*
 855                  * Check for any queued I/O requests that require an
 856                  * allocated slot.
 857                  */
 858                 io_req = TAILQ_FIRST(&softc->incoming_queue);
 859                 if (io_req == NULL) {
 860                         xpt_release_ccb(start_ccb);
 861                         break;
 862                 }
 863                 TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
 864                 TAILQ_INSERT_TAIL(&softc->active_queue, io_req, links);
 865                 /*
 866                  * Merge the user's CCB into the allocated CCB.
 867                  */
 868                 xpt_merge_ccb(start_ccb, &io_req->ccb);
 869                 start_ccb->ccb_h.ccb_type = PASS_CCB_QUEUED_IO;
 870                 start_ccb->ccb_h.ccb_ioreq = io_req;
 871                 start_ccb->ccb_h.cbfcnp = passdone;
 872                 io_req->alloced_ccb = start_ccb;
 873                 binuptime(&io_req->start_time);
 874                 devstat_start_transaction(softc->device_stats,
 875                                           &io_req->start_time);
 876
 877                 xpt_action(start_ccb);
 878
 879                 /*
 880                  * If we have any more I/O waiting, schedule ourselves again.
 881                  */
 882                 if (!TAILQ_EMPTY(&softc->incoming_queue))
 883                         xpt_schedule(periph, CAM_PRIORITY_NORMAL);
 884                 break;
 885         }
 886         default:
 887                 break;
 888         }
 889 }
 890
 891 static void
 892 passdone(struct cam_periph *periph, union ccb *done_ccb)
 893 {
 894         struct pass_softc *softc;
 895         struct ccb_scsiio *csio;
 896
 897         softc = (struct pass_softc *)periph->softc;
 898
 899         cam_periph_assert(periph, MA_OWNED);
 900
 901         csio = &done_ccb->csio;
 902         switch (csio->ccb_h.ccb_type) {
 903         case PASS_CCB_QUEUED_IO: {
 904                 struct pass_io_req *io_req;
 905
 906                 io_req = done_ccb->ccb_h.ccb_ioreq;
 907 #if 0
 908                 xpt_print(periph->path, "%s: called for user CCB %p\n",
 909                           __func__, io_req->user_ccb_ptr);
 910 #endif
 911                 if (((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
 912                  && (done_ccb->ccb_h.flags & CAM_PASS_ERR_RECOVER)
 913                  && ((io_req->flags & PASS_IO_ABANDONED) == 0)) {
 914                         int error;
 915
 916                         error = passerror(done_ccb, CAM_RETRY_SELTO,
 917                                           SF_RETRY_UA | SF_NO_PRINT);
 918
 919                         if (error == ERESTART) {
 920                                 /*
 921                                  * A retry was scheduled, so
 922                                  * just return.
 923                                  */
 924                                 return;
 925                         }
 926                 }
 927
 928                 /*
 929                  * Copy the allocated CCB contents back to the malloced CCB
 930                  * so we can give status back to the user when he requests it.
 931                  */
 932                 bcopy(done_ccb, &io_req->ccb, sizeof(*done_ccb));
 933
 934                 /*
 935                  * Log data/transaction completion with devstat(9).
 936                  */
 937                 switch (done_ccb->ccb_h.func_code) {
 938                 case XPT_SCSI_IO:
 939                         devstat_end_transaction(softc->device_stats,
 940                             done_ccb->csio.dxfer_len - done_ccb->csio.resid,
 941                             done_ccb->csio.tag_action & 0x3,
 942                             ((done_ccb->ccb_h.flags & CAM_DIR_MASK) ==
 943                             CAM_DIR_NONE) ? DEVSTAT_NO_DATA :
 944                             (done_ccb->ccb_h.flags & CAM_DIR_OUT) ?
 945                             DEVSTAT_WRITE : DEVSTAT_READ, NULL,
 946                             &io_req->start_time);
 947                         break;
 948                 case XPT_ATA_IO:
 949                         devstat_end_transaction(softc->device_stats,
 950                             done_ccb->ataio.dxfer_len - done_ccb->ataio.resid,
 951                             done_ccb->ataio.tag_action & 0x3,
 952                             ((done_ccb->ccb_h.flags & CAM_DIR_MASK) ==
 953                             CAM_DIR_NONE) ? DEVSTAT_NO_DATA :
 954                             (done_ccb->ccb_h.flags & CAM_DIR_OUT) ?
 955                             DEVSTAT_WRITE : DEVSTAT_READ, NULL,
 956                             &io_req->start_time);
 957                         break;
 958                 case XPT_SMP_IO:
 959                         /*
 960                          * XXX KDM this isn't quite right, but there isn't
 961                          * currently an easy way to represent a bidirectional
 962                          * transfer in devstat.  The only way to do it
 963                          * and have the byte counts come out right would
 964                          * mean that we would have to record two
 965                          * transactions, one for the request and one for the
 966                          * response.  For now, so that we report something,
 967                          * just treat the entire thing as a read.
 968                          */
 969                         devstat_end_transaction(softc->device_stats,
 970                             done_ccb->smpio.smp_request_len +
 971                             done_ccb->smpio.smp_response_len,
 972                             DEVSTAT_TAG_SIMPLE, DEVSTAT_READ, NULL,
 973                             &io_req->start_time);
 974                         break;
 975                 default:
 976                         devstat_end_transaction(softc->device_stats, 0,
 977                             DEVSTAT_TAG_NONE, DEVSTAT_NO_DATA, NULL,
 978                             &io_req->start_time);
 979                         break;
 980                 }
 981
 982                 /*
 983                  * In the normal case, take the completed I/O off of the
 984                  * active queue and put it on the done queue.  Notitfy the
 985                  * user that we have a completed I/O.
 986                  */
 987                 if ((io_req->flags & PASS_IO_ABANDONED) == 0) {
 988                         TAILQ_REMOVE(&softc->active_queue, io_req, links);
 989                         TAILQ_INSERT_TAIL(&softc->done_queue, io_req, links);
 990                         selwakeuppri(&softc->read_select, PRIBIO);
 991                         KNOTE_LOCKED(&softc->read_select.si_note, 0);
 992                 } else {
 993                         /*
 994                          * In the case of an abandoned I/O (final close
 995                          * without fetching the I/O), take it off of the
 996                          * abandoned queue and free it.
 997                          */
 998                         TAILQ_REMOVE(&softc->abandoned_queue, io_req, links);
 999                         passiocleanup(softc, io_req);
1000                         uma_zfree(softc->pass_zone, io_req);
1001
1002                         /*
1003                          * Release the done_ccb here, since we may wind up
1004                          * freeing the peripheral when we decrement the
1005                          * reference count below.
1006                          */
1007                         xpt_release_ccb(done_ccb);
1008
1009                         /*
1010                          * If the abandoned queue is empty, we can release
1011                          * our reference to the periph since we won't have
1012                          * any more completions coming.
1013                          */
1014                         if ((TAILQ_EMPTY(&softc->abandoned_queue))
1015                          && (softc->flags & PASS_FLAG_ABANDONED_REF_SET)) {
1016                                 softc->flags &= ~PASS_FLAG_ABANDONED_REF_SET;
1017                                 cam_periph_release_locked(periph);
1018                         }
1019
1020                         /*
1021                          * We have already released the CCB, so we can
1022                          * return.
1023                          */
1024                         return;
1025                 }
1026                 break;
1027         }
1028         }
1029         xpt_release_ccb(done_ccb);
1030 }
1031
1032 static int
1033 passcreatezone(struct cam_periph *periph)
1034 {
1035         struct pass_softc *softc;
1036         int error;
1037
1038         error = 0;
1039         softc = (struct pass_softc *)periph->softc;
1040
1041         cam_periph_assert(periph, MA_OWNED);
1042         KASSERT(((softc->flags & PASS_FLAG_ZONE_VALID) == 0),
1043                 ("%s called when the pass(4) zone is valid!\n", __func__));
1044         KASSERT((softc->pass_zone == NULL),
1045                 ("%s called when the pass(4) zone is allocated!\n", __func__));
1046
1047         if ((softc->flags & PASS_FLAG_ZONE_INPROG) == 0) {
1048
1049                 /*
1050                  * We're the first context through, so we need to create
1051                  * the pass(4) UMA zone for I/O requests.
1052                  */
1053                 softc->flags |= PASS_FLAG_ZONE_INPROG;
1054
1055                 /*
1056                  * uma_zcreate() does a blocking (M_WAITOK) allocation,
1057                  * so we cannot hold a mutex while we call it.
1058                  */
1059                 cam_periph_unlock(periph);
1060
1061                 softc->pass_zone = uma_zcreate(softc->zone_name,
1062                     sizeof(struct pass_io_req), NULL, NULL, NULL, NULL,
1063                     /*align*/ 0, /*flags*/ 0);
1064
1065                 softc->pass_io_zone = uma_zcreate(softc->io_zone_name,
1066                     softc->io_zone_size, NULL, NULL, NULL, NULL,
1067                     /*align*/ 0, /*flags*/ 0);
1068
1069                 cam_periph_lock(periph);
1070
1071                 if ((softc->pass_zone == NULL)
1072                  || (softc->pass_io_zone == NULL)) {
1073                         if (softc->pass_zone == NULL)
1074                                 xpt_print(periph->path, "unable to allocate "
1075                                     "IO Req UMA zone\n");
1076                         else
1077                                 xpt_print(periph->path, "unable to allocate "
1078                                     "IO UMA zone\n");
1079                         softc->flags &= ~PASS_FLAG_ZONE_INPROG;
1080                         goto bailout;
1081                 }
1082
1083                 /*
1084                  * Set the flags appropriately and notify any other waiters.
1085                  */
1086                 softc->flags &= PASS_FLAG_ZONE_INPROG;
1087                 softc->flags |= PASS_FLAG_ZONE_VALID;
1088                 wakeup(&softc->pass_zone);
1089         } else {
1090                 /*
1091                  * In this case, the UMA zone has not yet been created, but
1092                  * another context is in the process of creating it.  We
1093                  * need to sleep until the creation is either done or has
1094                  * failed.
1095                  */
1096                 while ((softc->flags & PASS_FLAG_ZONE_INPROG)
1097                     && ((softc->flags & PASS_FLAG_ZONE_VALID) == 0)) {
1098                         error = msleep(&softc->pass_zone,
1099                                        cam_periph_mtx(periph), PRIBIO,
1100                                        "paszon", 0);
1101                         if (error != 0)
1102                                 goto bailout;
1103                 }
1104                 /*
1105                  * If the zone creation failed, no luck for the user.
1106                  */
1107                 if ((softc->flags & PASS_FLAG_ZONE_VALID) == 0){
1108                         error = ENOMEM;
1109                         goto bailout;
1110                 }
1111         }
1112 bailout:
1113         return (error);
1114 }
1115
1116 static void
1117 passiocleanup(struct pass_softc *softc, struct pass_io_req *io_req)
1118 {
1119         union ccb *ccb;
1120         u_int8_t **data_ptrs[CAM_PERIPH_MAXMAPS];
1121         int i, numbufs;
1122
1123         ccb = &io_req->ccb;
1124
1125         switch (ccb->ccb_h.func_code) {
1126         case XPT_DEV_MATCH:
1127                 numbufs = min(io_req->num_bufs, 2);
1128
1129                 if (numbufs == 1) {
1130                         data_ptrs[0] = (u_int8_t **)&ccb->cdm.matches;
1131                 } else {
1132                         data_ptrs[0] = (u_int8_t **)&ccb->cdm.patterns;
1133                         data_ptrs[1] = (u_int8_t **)&ccb->cdm.matches;
1134                 }
1135                 break;
1136         case XPT_SCSI_IO:
1137         case XPT_CONT_TARGET_IO:
1138                 data_ptrs[0] = &ccb->csio.data_ptr;
1139                 numbufs = min(io_req->num_bufs, 1);
1140                 break;
1141         case XPT_ATA_IO:
1142                 data_ptrs[0] = &ccb->ataio.data_ptr;
1143                 numbufs = min(io_req->num_bufs, 1);
1144                 break;
1145         case XPT_SMP_IO:
1146                 numbufs = min(io_req->num_bufs, 2);
1147                 data_ptrs[0] = &ccb->smpio.smp_request;
1148                 data_ptrs[1] = &ccb->smpio.smp_response;
1149                 break;
1150         case XPT_DEV_ADVINFO:
1151                 numbufs = min(io_req->num_bufs, 1);
1152                 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf;
1153                 break;
1154         default:
1155                 /* allow ourselves to be swapped once again */
1156                 return;
1157                 break; /* NOTREACHED */
1158         }
1159
1160         if (io_req->flags & PASS_IO_USER_SEG_MALLOC) {
1161                 free(io_req->user_segptr, M_SCSIPASS);
1162                 io_req->user_segptr = NULL;
1163         }
1164
1165         /*
1166          * We only want to free memory we malloced.
1167          */
1168         if (io_req->data_flags == CAM_DATA_VADDR) {
1169                 for (i = 0; i < io_req->num_bufs; i++) {
1170                         if (io_req->kern_bufs[i] == NULL)
1171                                 continue;
1172
1173                         free(io_req->kern_bufs[i], M_SCSIPASS);
1174                         io_req->kern_bufs[i] = NULL;
1175                 }
1176         } else if (io_req->data_flags == CAM_DATA_SG) {
1177                 for (i = 0; i < io_req->num_kern_segs; i++) {
1178                         if ((uint8_t *)(uintptr_t)
1179                             io_req->kern_segptr[i].ds_addr == NULL)
1180                                 continue;
1181
1182                         uma_zfree(softc->pass_io_zone, (uint8_t *)(uintptr_t)
1183                             io_req->kern_segptr[i].ds_addr);
1184                         io_req->kern_segptr[i].ds_addr = 0;
1185                 }
1186         }
1187
1188         if (io_req->flags & PASS_IO_KERN_SEG_MALLOC) {
1189                 free(io_req->kern_segptr, M_SCSIPASS);
1190                 io_req->kern_segptr = NULL;
1191         }
1192
1193         if (io_req->data_flags != CAM_DATA_PADDR) {
1194                 for (i = 0; i < numbufs; i++) {
1195                         /*
1196                          * Restore the user's buffer pointers to their
1197                          * previous values.
1198                          */
1199                         if (io_req->user_bufs[i] != NULL)
1200                                 *data_ptrs[i] = io_req->user_bufs[i];
1201                 }
1202         }
1203
1204 }
1205
1206 static int
1207 passcopysglist(struct cam_periph *periph, struct pass_io_req *io_req,
1208                ccb_flags direction)
1209 {
1210         bus_size_t kern_watermark, user_watermark, len_copied, len_to_copy;
1211         bus_dma_segment_t *user_sglist, *kern_sglist;
1212         int i, j, error;
1213
1214         error = 0;
1215         kern_watermark = 0;
1216         user_watermark = 0;
1217         len_to_copy = 0;
1218         len_copied = 0;
1219         user_sglist = io_req->user_segptr;
1220         kern_sglist = io_req->kern_segptr;
1221
1222         for (i = 0, j = 0; i < io_req->num_user_segs &&
1223              j < io_req->num_kern_segs;) {
1224                 uint8_t *user_ptr, *kern_ptr;
1225
1226                 len_to_copy = min(user_sglist[i].ds_len -user_watermark,
1227                     kern_sglist[j].ds_len - kern_watermark);
1228
1229                 user_ptr = (uint8_t *)(uintptr_t)user_sglist[i].ds_addr;
1230                 user_ptr = user_ptr + user_watermark;
1231                 kern_ptr = (uint8_t *)(uintptr_t)kern_sglist[j].ds_addr;
1232                 kern_ptr = kern_ptr + kern_watermark;
1233
1234                 user_watermark += len_to_copy;
1235                 kern_watermark += len_to_copy;
1236
1237                 if (!useracc(user_ptr, len_to_copy,
1238                     (direction == CAM_DIR_IN) ? VM_PROT_WRITE : VM_PROT_READ)) {
1239                         xpt_print(periph->path, "%s: unable to access user "
1240                                   "S/G list element %p len %zu\n", __func__,
1241                                   user_ptr, len_to_copy);
1242                         error = EFAULT;
1243                         goto bailout;
1244                 }
1245
1246                 if (direction == CAM_DIR_IN) {
1247                         error = copyout(kern_ptr, user_ptr, len_to_copy);
1248                         if (error != 0) {
1249                                 xpt_print(periph->path, "%s: copyout of %u "
1250                                           "bytes from %p to %p failed with "
1251                                           "error %d\n", __func__, len_to_copy,
1252                                           kern_ptr, user_ptr, error);
1253                                 goto bailout;
1254                         }
1255                 } else {
1256                         error = copyin(user_ptr, kern_ptr, len_to_copy);
1257                         if (error != 0) {
1258                                 xpt_print(periph->path, "%s: copyin of %u "
1259                                           "bytes from %p to %p failed with "
1260                                           "error %d\n", __func__, len_to_copy,
1261                                           user_ptr, kern_ptr, error);
1262                                 goto bailout;
1263                         }
1264                 }
1265
1266                 len_copied += len_to_copy;
1267
1268                 if (user_sglist[i].ds_len == user_watermark) {
1269                         i++;
1270                         user_watermark = 0;
1271                 }
1272
1273                 if (kern_sglist[j].ds_len == kern_watermark) {
1274                         j++;
1275                         kern_watermark = 0;
1276                 }
1277         }
1278
1279 bailout:
1280
1281         return (error);
1282 }
1283
1284 static int
1285 passmemsetup(struct cam_periph *periph, struct pass_io_req *io_req)
1286 {
1287         union ccb *ccb;
1288         struct pass_softc *softc;
1289         int numbufs, i;
1290         uint8_t **data_ptrs[CAM_PERIPH_MAXMAPS];
1291         uint32_t lengths[CAM_PERIPH_MAXMAPS];
1292         uint32_t dirs[CAM_PERIPH_MAXMAPS];
1293         uint32_t num_segs;
1294         uint16_t *seg_cnt_ptr;
1295         size_t maxmap;
1296         int error;
1297
1298         cam_periph_assert(periph, MA_NOTOWNED);
1299
1300         softc = periph->softc;
1301
1302         error = 0;
1303         ccb = &io_req->ccb;
1304         maxmap = 0;
1305         num_segs = 0;
1306         seg_cnt_ptr = NULL;
1307
1308         switch(ccb->ccb_h.func_code) {
1309         case XPT_DEV_MATCH:
1310                 if (ccb->cdm.match_buf_len == 0) {
1311                         printf("%s: invalid match buffer length 0\n", __func__);
1312                         return(EINVAL);
1313                 }
1314                 if (ccb->cdm.pattern_buf_len > 0) {
1315                         data_ptrs[0] = (u_int8_t **)&ccb->cdm.patterns;
1316                         lengths[0] = ccb->cdm.pattern_buf_len;
1317                         dirs[0] = CAM_DIR_OUT;
1318                         data_ptrs[1] = (u_int8_t **)&ccb->cdm.matches;
1319                         lengths[1] = ccb->cdm.match_buf_len;
1320                         dirs[1] = CAM_DIR_IN;
1321                         numbufs = 2;
1322                 } else {
1323                         data_ptrs[0] = (u_int8_t **)&ccb->cdm.matches;
1324                         lengths[0] = ccb->cdm.match_buf_len;
1325                         dirs[0] = CAM_DIR_IN;
1326                         numbufs = 1;
1327                 }
1328                 io_req->data_flags = CAM_DATA_VADDR;
1329                 break;
1330         case XPT_SCSI_IO:
1331         case XPT_CONT_TARGET_IO:
1332                 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE)
1333                         return(0);
1334
1335                 /*
1336                  * The user shouldn't be able to supply a bio.
1337                  */
1338                 if ((ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO)
1339                         return (EINVAL);
1340
1341                 io_req->data_flags = ccb->ccb_h.flags & CAM_DATA_MASK;
1342
1343                 data_ptrs[0] = &ccb->csio.data_ptr;
1344                 lengths[0] = ccb->csio.dxfer_len;
1345                 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK;
1346                 num_segs = ccb->csio.sglist_cnt;
1347                 seg_cnt_ptr = &ccb->csio.sglist_cnt;
1348                 numbufs = 1;
1349                 maxmap = softc->maxio;
1350                 break;
1351         case XPT_ATA_IO:
1352                 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE)
1353                         return(0);
1354
1355                 /*
1356                  * We only support a single virtual address for ATA I/O.
1357                  */
1358                 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR)
1359                         return (EINVAL);
1360
1361                 io_req->data_flags = CAM_DATA_VADDR;
1362
1363                 data_ptrs[0] = &ccb->ataio.data_ptr;
1364                 lengths[0] = ccb->ataio.dxfer_len;
1365                 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK;
1366                 numbufs = 1;
1367                 maxmap = softc->maxio;
1368                 break;
1369         case XPT_SMP_IO:
1370                 io_req->data_flags = CAM_DATA_VADDR;
1371
1372                 data_ptrs[0] = &ccb->smpio.smp_request;
1373                 lengths[0] = ccb->smpio.smp_request_len;
1374                 dirs[0] = CAM_DIR_OUT;
1375                 data_ptrs[1] = &ccb->smpio.smp_response;
1376                 lengths[1] = ccb->smpio.smp_response_len;
1377                 dirs[1] = CAM_DIR_IN;
1378                 numbufs = 2;
1379                 maxmap = softc->maxio;
1380                 break;
1381         case XPT_DEV_ADVINFO:
1382                 if (ccb->cdai.bufsiz == 0)
1383                         return (0);
1384
1385                 io_req->data_flags = CAM_DATA_VADDR;
1386
1387                 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf;
1388                 lengths[0] = ccb->cdai.bufsiz;
1389                 dirs[0] = CAM_DIR_IN;
1390                 numbufs = 1;
1391                 break;
1392         default:
1393                 return(EINVAL);
1394                 break; /* NOTREACHED */
1395         }
1396
1397         io_req->num_bufs = numbufs;
1398
1399         /*
1400          * If there is a maximum, check to make sure that the user's
1401          * request fits within the limit.  In general, we should only have
1402          * a maximum length for requests that go to hardware.  Otherwise it
1403          * is whatever we're able to malloc.
1404          */
1405         for (i = 0; i < numbufs; i++) {
1406                 io_req->user_bufs[i] = *data_ptrs[i];
1407                 io_req->dirs[i] = dirs[i];
1408                 io_req->lengths[i] = lengths[i];
1409
1410                 if (maxmap == 0)
1411                         continue;
1412
1413                 if (lengths[i] <= maxmap)
1414                         continue;
1415
1416                 xpt_print(periph->path, "%s: data length %u > max allowed %u "
1417                           "bytes\n", __func__, lengths[i], maxmap);
1418                 error = EINVAL;
1419                 goto bailout;
1420         }
1421
1422         switch (io_req->data_flags) {
1423         case CAM_DATA_VADDR:
1424                 /* Map or copy the buffer into kernel address space */
1425                 for (i = 0; i < numbufs; i++) {
1426                         uint8_t *tmp_buf;
1427
1428                         /*
1429                          * If for some reason no length is specified, we
1430                          * don't need to allocate anything.
1431                          */
1432                         if (io_req->lengths[i] == 0)
1433                                 continue;
1434
1435                         /*
1436                          * Make sure that the user's buffer is accessible
1437                          * to that process.
1438                          */
1439                         if (!useracc(io_req->user_bufs[i], io_req->lengths[i],
1440                             (io_req->dirs[i] == CAM_DIR_IN) ? VM_PROT_WRITE :
1441                              VM_PROT_READ)) {
1442                                 xpt_print(periph->path, "%s: user address %p "
1443                                     "length %u is not accessible\n", __func__,
1444                                     io_req->user_bufs[i], io_req->lengths[i]);
1445                                 error = EFAULT;
1446                                 goto bailout;
1447                         }
1448
1449                         tmp_buf = malloc(lengths[i], M_SCSIPASS,
1450                                          M_WAITOK | M_ZERO);
1451                         io_req->kern_bufs[i] = tmp_buf;
1452                         *data_ptrs[i] = tmp_buf;
1453
1454 #if 0
1455                         xpt_print(periph->path, "%s: malloced %p len %u, user "
1456                                   "buffer %p, operation: %s\n", __func__,
1457                                   tmp_buf, lengths[i], io_req->user_bufs[i],
1458                                   (dirs[i] == CAM_DIR_IN) ? "read" : "write");
1459 #endif
1460                         /*
1461                          * We only need to copy in if the user is writing.
1462                          */
1463                         if (dirs[i] != CAM_DIR_OUT)
1464                                 continue;
1465
1466                         error = copyin(io_req->user_bufs[i],
1467                                        io_req->kern_bufs[i], lengths[i]);
1468                         if (error != 0) {
1469                                 xpt_print(periph->path, "%s: copy of user "
1470                                           "buffer from %p to %p failed with "
1471                                           "error %d\n", __func__,
1472                                           io_req->user_bufs[i],
1473                                           io_req->kern_bufs[i], error);
1474                                 goto bailout;
1475                         }
1476                 }
1477                 break;
1478         case CAM_DATA_PADDR:
1479                 /* Pass down the pointer as-is */
1480                 break;
1481         case CAM_DATA_SG: {
1482                 size_t sg_length, size_to_go, alloc_size;
1483                 uint32_t num_segs_needed;
1484
1485                 /*
1486                  * Copy the user S/G list in, and then copy in the
1487                  * individual segments.
1488                  */
1489                 /*
1490                  * We shouldn't see this, but check just in case.
1491                  */
1492                 if (numbufs != 1) {
1493                         xpt_print(periph->path, "%s: cannot currently handle "
1494                                   "more than one S/G list per CCB\n", __func__);
1495                         error = EINVAL;
1496                         goto bailout;
1497                 }
1498
1499                 /*
1500                  * We have to have at least one segment.
1501                  */
1502                 if (num_segs == 0) {
1503                         xpt_print(periph->path, "%s: CAM_DATA_SG flag set, "
1504                                   "but sglist_cnt=0!\n", __func__);
1505                         error = EINVAL;
1506                         goto bailout;
1507                 }
1508
1509                 /*
1510                  * Make sure the user specified the total length and didn't
1511                  * just leave it to us to decode the S/G list.
1512                  */
1513                 if (lengths[0] == 0) {
1514                         xpt_print(periph->path, "%s: no dxfer_len specified, "
1515                                   "but CAM_DATA_SG flag is set!\n", __func__);
1516                         error = EINVAL;
1517                         goto bailout;
1518                 }
1519
1520                 /*
1521                  * We allocate buffers in io_zone_size increments for an
1522                  * S/G list.  This will generally be MAXPHYS.
1523                  */
1524                 if (lengths[0] <= softc->io_zone_size)
1525                         num_segs_needed = 1;
1526                 else {
1527                         num_segs_needed = lengths[0] / softc->io_zone_size;
1528                         if ((lengths[0] % softc->io_zone_size) != 0)
1529                                 num_segs_needed++;
1530                 }
1531
1532                 /* Figure out the size of the S/G list */
1533                 sg_length = num_segs * sizeof(bus_dma_segment_t);
1534                 io_req->num_user_segs = num_segs;
1535                 io_req->num_kern_segs = num_segs_needed;
1536
1537                 /* Save the user's S/G list pointer for later restoration */
1538                 io_req->user_bufs[0] = *data_ptrs[0];
1539
1540                 /*
1541                  * If we have enough segments allocated by default to handle
1542                  * the length of the user's S/G list,
1543                  */
1544                 if (num_segs > PASS_MAX_SEGS) {
1545                         io_req->user_segptr = malloc(sizeof(bus_dma_segment_t) *
1546                             num_segs, M_SCSIPASS, M_WAITOK | M_ZERO);
1547                         io_req->flags |= PASS_IO_USER_SEG_MALLOC;
1548                 } else
1549                         io_req->user_segptr = io_req->user_segs;
1550
1551                 if (!useracc(*data_ptrs[0], sg_length, VM_PROT_READ)) {
1552                         xpt_print(periph->path, "%s: unable to access user "
1553                                   "S/G list at %p\n", __func__, *data_ptrs[0]);
1554                         error = EFAULT;
1555                         goto bailout;
1556                 }
1557
1558                 error = copyin(*data_ptrs[0], io_req->user_segptr, sg_length);
1559                 if (error != 0) {
1560                         xpt_print(periph->path, "%s: copy of user S/G list "
1561                                   "from %p to %p failed with error %d\n",
1562                                   __func__, *data_ptrs[0], io_req->user_segptr,
1563                                   error);
1564                         goto bailout;
1565                 }
1566
1567                 if (num_segs_needed > PASS_MAX_SEGS) {
1568                         io_req->kern_segptr = malloc(sizeof(bus_dma_segment_t) *
1569                             num_segs_needed, M_SCSIPASS, M_WAITOK | M_ZERO);
1570                         io_req->flags |= PASS_IO_KERN_SEG_MALLOC;
1571                 } else {
1572                         io_req->kern_segptr = io_req->kern_segs;
1573                 }
1574
1575                 /*
1576                  * Allocate the kernel S/G list.
1577                  */
1578                 for (size_to_go = lengths[0], i = 0;
1579                      size_to_go > 0 && i < num_segs_needed;
1580                      i++, size_to_go -= alloc_size) {
1581                         uint8_t *kern_ptr;
1582
1583                         alloc_size = min(size_to_go, softc->io_zone_size);
1584                         kern_ptr = uma_zalloc(softc->pass_io_zone, M_WAITOK);
1585                         io_req->kern_segptr[i].ds_addr =
1586                             (bus_addr_t)(uintptr_t)kern_ptr;
1587                         io_req->kern_segptr[i].ds_len = alloc_size;
1588                 }
1589                 if (size_to_go > 0) {
1590                         printf("%s: size_to_go = %zu, software error!\n",
1591                                __func__, size_to_go);
1592                         error = EINVAL;
1593                         goto bailout;
1594                 }
1595
1596                 *data_ptrs[0] = (uint8_t *)io_req->kern_segptr;
1597                 *seg_cnt_ptr = io_req->num_kern_segs;
1598
1599                 /*
1600                  * We only need to copy data here if the user is writing.
1601                  */
1602                 if (dirs[0] == CAM_DIR_OUT)
1603                         error = passcopysglist(periph, io_req, dirs[0]);
1604                 break;
1605         }
1606         case CAM_DATA_SG_PADDR: {
1607                 size_t sg_length;
1608
1609                 /*
1610                  * We shouldn't see this, but check just in case.
1611                  */
1612                 if (numbufs != 1) {
1613                         printf("%s: cannot currently handle more than one "
1614                                "S/G list per CCB\n", __func__);
1615                         error = EINVAL;
1616                         goto bailout;
1617                 }
1618
1619                 /*
1620                  * We have to have at least one segment.
1621                  */
1622                 if (num_segs == 0) {
1623                         xpt_print(periph->path, "%s: CAM_DATA_SG_PADDR flag "
1624                                   "set, but sglist_cnt=0!\n", __func__);
1625                         error = EINVAL;
1626                         goto bailout;
1627                 }
1628
1629                 /*
1630                  * Make sure the user specified the total length and didn't
1631                  * just leave it to us to decode the S/G list.
1632                  */
1633                 if (lengths[0] == 0) {
1634                         xpt_print(periph->path, "%s: no dxfer_len specified, "
1635                                   "but CAM_DATA_SG flag is set!\n", __func__);
1636                         error = EINVAL;
1637                         goto bailout;
1638                 }
1639
1640                 /* Figure out the size of the S/G list */
1641                 sg_length = num_segs * sizeof(bus_dma_segment_t);
1642                 io_req->num_user_segs = num_segs;
1643                 io_req->num_kern_segs = io_req->num_user_segs;
1644
1645                 /* Save the user's S/G list pointer for later restoration */
1646                 io_req->user_bufs[0] = *data_ptrs[0];
1647
1648                 if (num_segs > PASS_MAX_SEGS) {
1649                         io_req->user_segptr = malloc(sizeof(bus_dma_segment_t) *
1650                             num_segs, M_SCSIPASS, M_WAITOK | M_ZERO);
1651                         io_req->flags |= PASS_IO_USER_SEG_MALLOC;
1652                 } else
1653                         io_req->user_segptr = io_req->user_segs;
1654
1655                 io_req->kern_segptr = io_req->user_segptr;
1656
1657                 error = copyin(*data_ptrs[0], io_req->user_segptr, sg_length);
1658                 if (error != 0) {
1659                         xpt_print(periph->path, "%s: copy of user S/G list "
1660                                   "from %p to %p failed with error %d\n",
1661                                   __func__, *data_ptrs[0], io_req->user_segptr,
1662                                   error);
1663                         goto bailout;
1664                 }
1665                 break;
1666         }
1667         default:
1668         case CAM_DATA_BIO:
1669                 /*
1670                  * A user shouldn't be attaching a bio to the CCB.  It
1671                  * isn't a user-accessible structure.
1672                  */
1673                 error = EINVAL;
1674                 break;
1675         }
1676
1677 bailout:
1678         if (error != 0)
1679                 passiocleanup(softc, io_req);
1680
1681         return (error);
1682 }
1683
1684 static int
1685 passmemdone(struct cam_periph *periph, struct pass_io_req *io_req)
1686 {
1687         struct pass_softc *softc;
1688         union ccb *ccb;
1689         int error;
1690         int i;
1691
1692         error = 0;
1693         softc = (struct pass_softc *)periph->softc;
1694         ccb = &io_req->ccb;
1695
1696         switch (io_req->data_flags) {
1697         case CAM_DATA_VADDR:
1698                 /*
1699                  * Copy back to the user buffer if this was a read.
1700                  */
1701                 for (i = 0; i < io_req->num_bufs; i++) {
1702                         if (io_req->dirs[i] != CAM_DIR_IN)
1703                                 continue;
1704
1705                         error = copyout(io_req->kern_bufs[i],
1706                             io_req->user_bufs[i], io_req->lengths[i]);
1707                         if (error != 0) {
1708                                 xpt_print(periph->path, "Unable to copy %u "
1709                                           "bytes from %p to user address %p\n",
1710                                           io_req->lengths[i],
1711                                           io_req->kern_bufs[i],
1712                                           io_req->user_bufs[i]);
1713                                 goto bailout;
1714                         }
1715
1716                 }
1717                 break;
1718         case CAM_DATA_PADDR:
1719                 /* Do nothing.  The pointer is a physical address already */
1720                 break;
1721         case CAM_DATA_SG:
1722                 /*
1723                  * Copy back to the user buffer if this was a read.
1724                  * Restore the user's S/G list buffer pointer.
1725                  */
1726                 if (io_req->dirs[0] == CAM_DIR_IN)
1727                         error = passcopysglist(periph, io_req, io_req->dirs[0]);
1728                 break;
1729         case CAM_DATA_SG_PADDR:
1730                 /*
1731                  * Restore the user's S/G list buffer pointer.  No need to
1732                  * copy.
1733                  */
1734                 break;
1735         default:
1736         case CAM_DATA_BIO:
1737                 error = EINVAL;
1738                 break;
1739         }
1740
1741 bailout:
1742         /*
1743          * Reset the user's pointers to their original values and free
1744          * allocated memory.
1745          */
1746         passiocleanup(softc, io_req);
1747
1748         return (error);
1749 }
1750
1751 static int
1752 passioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
1753 {
1754         int error;
1755
1756         if ((error = passdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
1757                 error = cam_compat_ioctl(dev, cmd, addr, flag, td, passdoioctl);
1758         }
1759         return (error);
1760 }
1761
1762 static int
1763 passdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
1764 {
1765         struct  cam_periph *periph;
1766         struct  pass_softc *softc;
1767         int     error;
1768         uint32_t priority;
1769
1770         periph = (struct cam_periph *)dev->si_drv1;
1771         cam_periph_lock(periph);
1772         softc = (struct pass_softc *)periph->softc;
1773
1774         error = 0;
1775
1776         switch (cmd) {
1777
1778         case CAMIOCOMMAND:
1779         {
1780                 union ccb *inccb;
1781                 union ccb *ccb;
1782                 int ccb_malloced;
1783
1784                 inccb = (union ccb *)addr;
1785
1786                 /*
1787                  * Some CCB types, like scan bus and scan lun can only go
1788                  * through the transport layer device.
1789                  */
1790                 if (inccb->ccb_h.func_code & XPT_FC_XPT_ONLY) {
1791                         xpt_print(periph->path, "CCB function code %#x is "
1792                             "restricted to the XPT device\n",
1793                             inccb->ccb_h.func_code);
1794                         error = ENODEV;
1795                         break;
1796                 }
1797
1798                 /* Compatibility for RL/priority-unaware code. */
1799                 priority = inccb->ccb_h.pinfo.priority;
1800                 if (priority <= CAM_PRIORITY_OOB)
1801                     priority += CAM_PRIORITY_OOB + 1;
1802
1803                 /*
1804                  * Non-immediate CCBs need a CCB from the per-device pool
1805                  * of CCBs, which is scheduled by the transport layer.
1806                  * Immediate CCBs and user-supplied CCBs should just be
1807                  * malloced.
1808                  */
1809                 if ((inccb->ccb_h.func_code & XPT_FC_QUEUED)
1810                  && ((inccb->ccb_h.func_code & XPT_FC_USER_CCB) == 0)) {
1811                         ccb = cam_periph_getccb(periph, priority);
1812                         ccb_malloced = 0;
1813                 } else {
1814                         ccb = xpt_alloc_ccb_nowait();
1815
1816                         if (ccb != NULL)
1817                                 xpt_setup_ccb(&ccb->ccb_h, periph->path,
1818                                               priority);
1819                         ccb_malloced = 1;
1820                 }
1821
1822                 if (ccb == NULL) {
1823                         xpt_print(periph->path, "unable to allocate CCB\n");
1824                         error = ENOMEM;
1825                         break;
1826                 }
1827
1828                 error = passsendccb(periph, ccb, inccb);
1829
1830                 if (ccb_malloced)
1831                         xpt_free_ccb(ccb);
1832                 else
1833                         xpt_release_ccb(ccb);
1834
1835                 break;
1836         }
1837         case CAMIOQUEUE:
1838         {
1839                 struct pass_io_req *io_req;
1840                 union ccb **user_ccb, *ccb;
1841                 xpt_opcode fc;
1842
1843                 if ((softc->flags & PASS_FLAG_ZONE_VALID) == 0) {
1844                         error = passcreatezone(periph);
1845                         if (error != 0)
1846                                 goto bailout;
1847                 }
1848
1849                 /*
1850                  * We're going to do a blocking allocation for this I/O
1851                  * request, so we have to drop the lock.
1852                  */
1853                 cam_periph_unlock(periph);
1854
1855                 io_req = uma_zalloc(softc->pass_zone, M_WAITOK | M_ZERO);
1856                 ccb = &io_req->ccb;
1857                 user_ccb = (union ccb **)addr;
1858
1859                 /*
1860                  * Unlike the CAMIOCOMMAND ioctl above, we only have a
1861                  * pointer to the user's CCB, so we have to copy the whole
1862                  * thing in to a buffer we have allocated (above) instead
1863                  * of allowing the ioctl code to malloc a buffer and copy
1864                  * it in.
1865                  *
1866                  * This is an advantage for this asynchronous interface,
1867                  * since we don't want the memory to get freed while the
1868                  * CCB is outstanding.
1869                  */
1870 #if 0
1871                 xpt_print(periph->path, "Copying user CCB %p to "
1872                           "kernel address %p\n", *user_ccb, ccb);
1873 #endif
1874                 error = copyin(*user_ccb, ccb, sizeof(*ccb));
1875                 if (error != 0) {
1876                         xpt_print(periph->path, "Copy of user CCB %p to "
1877                                   "kernel address %p failed with error %d\n",
1878                                   *user_ccb, ccb, error);
1879                         uma_zfree(softc->pass_zone, io_req);
1880                         cam_periph_lock(periph);
1881                         break;
1882                 }
1883
1884                 /*
1885                  * Some CCB types, like scan bus and scan lun can only go
1886                  * through the transport layer device.
1887                  */
1888                 if (ccb->ccb_h.func_code & XPT_FC_XPT_ONLY) {
1889                         xpt_print(periph->path, "CCB function code %#x is "
1890                             "restricted to the XPT device\n",
1891                             ccb->ccb_h.func_code);
1892                         uma_zfree(softc->pass_zone, io_req);
1893                         cam_periph_lock(periph);
1894                         error = ENODEV;
1895                         break;
1896                 }
1897
1898                 /*
1899                  * Save the user's CCB pointer as well as his linked list
1900                  * pointers and peripheral private area so that we can
1901                  * restore these later.
1902                  */
1903                 io_req->user_ccb_ptr = *user_ccb;
1904                 io_req->user_periph_links = ccb->ccb_h.periph_links;
1905                 io_req->user_periph_priv = ccb->ccb_h.periph_priv;
1906
1907                 /*
1908                  * Now that we've saved the user's values, we can set our
1909                  * own peripheral private entry.
1910                  */
1911                 ccb->ccb_h.ccb_ioreq = io_req;
1912
1913                 /* Compatibility for RL/priority-unaware code. */
1914                 priority = ccb->ccb_h.pinfo.priority;
1915                 if (priority <= CAM_PRIORITY_OOB)
1916                     priority += CAM_PRIORITY_OOB + 1;
1917
1918                 /*
1919                  * Setup fields in the CCB like the path and the priority.
1920                  * The path in particular cannot be done in userland, since
1921                  * it is a pointer to a kernel data structure.
1922                  */
1923                 xpt_setup_ccb_flags(&ccb->ccb_h, periph->path, priority,
1924                                     ccb->ccb_h.flags);
1925
1926                 /*
1927                  * Setup our done routine.  There is no way for the user to
1928                  * have a valid pointer here.
1929                  */
1930                 ccb->ccb_h.cbfcnp = passdone;
1931
1932                 fc = ccb->ccb_h.func_code;
1933                 /*
1934                  * If this function code has memory that can be mapped in
1935                  * or out, we need to call passmemsetup().
1936                  */
1937                 if ((fc == XPT_SCSI_IO) || (fc == XPT_ATA_IO)
1938                  || (fc == XPT_SMP_IO) || (fc == XPT_DEV_MATCH)
1939                  || (fc == XPT_DEV_ADVINFO)) {
1940                         error = passmemsetup(periph, io_req);
1941                         if (error != 0) {
1942                                 uma_zfree(softc->pass_zone, io_req);
1943                                 cam_periph_lock(periph);
1944                                 break;
1945                         }
1946                 } else
1947                         io_req->mapinfo.num_bufs_used = 0;
1948
1949                 cam_periph_lock(periph);
1950
1951                 /*
1952                  * Everything goes on the incoming queue initially.
1953                  */
1954                 TAILQ_INSERT_TAIL(&softc->incoming_queue, io_req, links);
1955
1956                 /*
1957                  * If the CCB is queued, and is not a user CCB, then
1958                  * we need to allocate a slot for it.  Call xpt_schedule()
1959                  * so that our start routine will get called when a CCB is
1960                  * available.
1961                  */
1962                 if ((fc & XPT_FC_QUEUED)
1963                  && ((fc & XPT_FC_USER_CCB) == 0)) {
1964                         xpt_schedule(periph, priority);
1965                         break;
1966                 }
1967
1968                 /*
1969                  * At this point, the CCB in question is either an
1970                  * immediate CCB (like XPT_DEV_ADVINFO) or it is a user CCB
1971                  * and therefore should be malloced, not allocated via a slot.
1972                  * Remove the CCB from the incoming queue and add it to the
1973                  * active queue.
1974                  */
1975                 TAILQ_REMOVE(&softc->incoming_queue, io_req, links);
1976                 TAILQ_INSERT_TAIL(&softc->active_queue, io_req, links);
1977
1978                 xpt_action(ccb);
1979
1980                 /*
1981                  * If this is not a queued CCB (i.e. it is an immediate CCB),
1982                  * then it is already done.  We need to put it on the done
1983                  * queue for the user to fetch.
1984                  */
1985                 if ((fc & XPT_FC_QUEUED) == 0) {
1986                         TAILQ_REMOVE(&softc->active_queue, io_req, links);
1987                         TAILQ_INSERT_TAIL(&softc->done_queue, io_req, links);
1988                 }
1989                 break;
1990         }
1991         case CAMIOGET:
1992         {
1993                 union ccb **user_ccb;
1994                 struct pass_io_req *io_req;
1995                 int old_error;
1996
1997                 user_ccb = (union ccb **)addr;
1998                 old_error = 0;
1999
2000                 io_req = TAILQ_FIRST(&softc->done_queue);
2001                 if (io_req == NULL) {
2002                         error = ENOENT;
2003                         break;
2004                 }
2005
2006                 /*
2007                  * Remove the I/O from the done queue.
2008                  */
2009                 TAILQ_REMOVE(&softc->done_queue, io_req, links);
2010
2011                 /*
2012                  * We have to drop the lock during the copyout because the
2013                  * copyout can result in VM faults that require sleeping.
2014                  */
2015                 cam_periph_unlock(periph);
2016
2017                 /*
2018                  * Do any needed copies (e.g. for reads) and revert the
2019                  * pointers in the CCB back to the user's pointers.
2020                  */
2021                 error = passmemdone(periph, io_req);
2022
2023                 old_error = error;
2024
2025                 io_req->ccb.ccb_h.periph_links = io_req->user_periph_links;
2026                 io_req->ccb.ccb_h.periph_priv = io_req->user_periph_priv;
2027
2028 #if 0
2029                 xpt_print(periph->path, "Copying to user CCB %p from "
2030                           "kernel address %p\n", *user_ccb, &io_req->ccb);
2031 #endif
2032
2033                 error = copyout(&io_req->ccb, *user_ccb, sizeof(union ccb));
2034                 if (error != 0) {
2035                         xpt_print(periph->path, "Copy to user CCB %p from "
2036                                   "kernel address %p failed with error %d\n",
2037                                   *user_ccb, &io_req->ccb, error);
2038                 }
2039
2040                 /*
2041                  * Prefer the first error we got back, and make sure we
2042                  * don't overwrite bad status with good.
2043                  */
2044                 if (old_error != 0)
2045                         error = old_error;
2046
2047                 cam_periph_lock(periph);
2048
2049                 /*
2050                  * At this point, if there was an error, we could potentially
2051                  * re-queue the I/O and try again.  But why?  The error
2052                  * would almost certainly happen again.  We might as well
2053                  * not leak memory.
2054                  */
2055                 uma_zfree(softc->pass_zone, io_req);
2056                 break;
2057         }
2058         default:
2059                 error = cam_periph_ioctl(periph, cmd, addr, passerror);
2060                 break;
2061         }
2062
2063 bailout:
2064         cam_periph_unlock(periph);
2065
2066         return(error);
2067 }
2068
2069 static int
2070 passpoll(struct cdev *dev, int poll_events, struct thread *td)
2071 {
2072         struct cam_periph *periph;
2073         struct pass_softc *softc;
2074         int revents;
2075
2076         periph = (struct cam_periph *)dev->si_drv1;
2077         softc = (struct pass_softc *)periph->softc;
2078
2079         revents = poll_events & (POLLOUT | POLLWRNORM);
2080         if ((poll_events & (POLLIN | POLLRDNORM)) != 0) {
2081                 cam_periph_lock(periph);
2082
2083                 if (!TAILQ_EMPTY(&softc->done_queue)) {
2084                         revents |= poll_events & (POLLIN | POLLRDNORM);
2085                 }
2086                 cam_periph_unlock(periph);
2087                 if (revents == 0)
2088                         selrecord(td, &softc->read_select);
2089         }
2090
2091         return (revents);
2092 }
2093
2094 static int
2095 passkqfilter(struct cdev *dev, struct knote *kn)
2096 {
2097         struct cam_periph *periph;
2098         struct pass_softc *softc;
2099
2100         periph = (struct cam_periph *)dev->si_drv1;
2101         softc = (struct pass_softc *)periph->softc;
2102
2103         kn->kn_hook = (caddr_t)periph;
2104         kn->kn_fop = &passread_filtops;
2105         knlist_add(&softc->read_select.si_note, kn, 0);
2106
2107         return (0);
2108 }
2109
2110 static void
2111 passreadfiltdetach(struct knote *kn)
2112 {
2113         struct cam_periph *periph;
2114         struct pass_softc *softc;
2115
2116         periph = (struct cam_periph *)kn->kn_hook;
2117         softc = (struct pass_softc *)periph->softc;
2118
2119         knlist_remove(&softc->read_select.si_note, kn, 0);
2120 }
2121
2122 static int
2123 passreadfilt(struct knote *kn, long hint)
2124 {
2125         struct cam_periph *periph;
2126         struct pass_softc *softc;
2127         int retval;
2128
2129         periph = (struct cam_periph *)kn->kn_hook;
2130         softc = (struct pass_softc *)periph->softc;
2131
2132         cam_periph_assert(periph, MA_OWNED);
2133
2134         if (TAILQ_EMPTY(&softc->done_queue))
2135                 retval = 0;
2136         else
2137                 retval = 1;
2138
2139         return (retval);
2140 }
2141
2142 /*
2143  * Generally, "ccb" should be the CCB supplied by the kernel.  "inccb"
2144  * should be the CCB that is copied in from the user.
2145  */
2146 static int
2147 passsendccb(struct cam_periph *periph, union ccb *ccb, union ccb *inccb)
2148 {
2149         struct pass_softc *softc;
2150         struct cam_periph_map_info mapinfo;
2151         xpt_opcode fc;
2152         int error;
2153
2154         softc = (struct pass_softc *)periph->softc;
2155
2156         /*
2157          * There are some fields in the CCB header that need to be
2158          * preserved, the rest we get from the user.
2159          */
2160         xpt_merge_ccb(ccb, inccb);
2161
2162         /*
2163          */
2164         ccb->ccb_h.cbfcnp = passdone;
2165
2166         /*
2167          * Let cam_periph_mapmem do a sanity check on the data pointer format.
2168          * Even if no data transfer is needed, it's a cheap check and it
2169          * simplifies the code.
2170          */
2171         fc = ccb->ccb_h.func_code;
2172         if ((fc == XPT_SCSI_IO) || (fc == XPT_ATA_IO) || (fc == XPT_SMP_IO)
2173          || (fc == XPT_DEV_MATCH) || (fc == XPT_DEV_ADVINFO)) {
2174                 bzero(&mapinfo, sizeof(mapinfo));
2175
2176                 /*
2177                  * cam_periph_mapmem calls into proc and vm functions that can
2178                  * sleep as well as trigger I/O, so we can't hold the lock.
2179                  * Dropping it here is reasonably safe.
2180                  */
2181                 cam_periph_unlock(periph);
2182                 error = cam_periph_mapmem(ccb, &mapinfo, softc->maxio);
2183                 cam_periph_lock(periph);
2184
2185                 /*
2186                  * cam_periph_mapmem returned an error, we can't continue.
2187                  * Return the error to the user.
2188                  */
2189                 if (error)
2190                         return(error);
2191         } else
2192                 /* Ensure that the unmap call later on is a no-op. */
2193                 mapinfo.num_bufs_used = 0;
2194
2195         /*
2196          * If the user wants us to perform any error recovery, then honor
2197          * that request.  Otherwise, it's up to the user to perform any
2198          * error recovery.
2199          */
2200         cam_periph_runccb(ccb, passerror, /* cam_flags */ CAM_RETRY_SELTO,
2201             /* sense_flags */ ((ccb->ccb_h.flags & CAM_PASS_ERR_RECOVER) ?
2202              SF_RETRY_UA : SF_NO_RECOVERY) | SF_NO_PRINT,
2203             softc->device_stats);
2204
2205         cam_periph_unmapmem(ccb, &mapinfo);
2206
2207         ccb->ccb_h.cbfcnp = NULL;
2208         ccb->ccb_h.periph_priv = inccb->ccb_h.periph_priv;
2209         bcopy(ccb, inccb, sizeof(union ccb));
2210
2211         return(0);
2212 }
2213
2214 static int
2215 passerror(union ccb *ccb, u_int32_t cam_flags, u_int32_t sense_flags)
2216 {
2217         struct cam_periph *periph;
2218         struct pass_softc *softc;
2219
2220         periph = xpt_path_periph(ccb->ccb_h.path);
2221         softc = (struct pass_softc *)periph->softc;
2222
2223         return(cam_periph_error(ccb, cam_flags, sense_flags,
2224                                  &softc->saved_ccb));
2225 }