sys/cam/cam_xpt.c

   1 /*-
   2  * Implementation of the Common Access Method Transport (XPT) layer.
   3  *
   4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
   5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification, immediately at the beginning of the file.
  14  * 2. The name of the author may not be used to endorse or promote products
  15  *    derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include "opt_printf.h"
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/bio.h>
  37 #include <sys/bus.h>
  38 #include <sys/systm.h>
  39 #include <sys/types.h>
  40 #include <sys/malloc.h>
  41 #include <sys/kernel.h>
  42 #include <sys/time.h>
  43 #include <sys/conf.h>
  44 #include <sys/fcntl.h>
  45 #include <sys/interrupt.h>
  46 #include <sys/proc.h>
  47 #include <sys/sbuf.h>
  48 #include <sys/smp.h>
  49 #include <sys/taskqueue.h>
  50
  51 #include <sys/lock.h>
  52 #include <sys/mutex.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/kthread.h>
  55
  56 #include <cam/cam.h>
  57 #include <cam/cam_ccb.h>
  58 #include <cam/cam_periph.h>
  59 #include <cam/cam_queue.h>
  60 #include <cam/cam_sim.h>
  61 #include <cam/cam_xpt.h>
  62 #include <cam/cam_xpt_sim.h>
  63 #include <cam/cam_xpt_periph.h>
  64 #include <cam/cam_xpt_internal.h>
  65 #include <cam/cam_debug.h>
  66 #include <cam/cam_compat.h>
  67
  68 #include <cam/scsi/scsi_all.h>
  69 #include <cam/scsi/scsi_message.h>
  70 #include <cam/scsi/scsi_pass.h>
  71
  72 #include <machine/md_var.h>     /* geometry translation */
  73 #include <machine/stdarg.h>     /* for xpt_print below */
  74
  75 #include "opt_cam.h"
  76
  77 /* Wild guess based on not wanting to grow the stack too much */
  78 #define XPT_PRINT_MAXLEN        512
  79 #ifdef PRINTF_BUFR_SIZE
  80 #define XPT_PRINT_LEN   PRINTF_BUFR_SIZE
  81 #else
  82 #define XPT_PRINT_LEN   128
  83 #endif
  84 _Static_assert(XPT_PRINT_LEN <= XPT_PRINT_MAXLEN, "XPT_PRINT_LEN is too large");
  85
  86 /*
  87  * This is the maximum number of high powered commands (e.g. start unit)
  88  * that can be outstanding at a particular time.
  89  */
  90 #ifndef CAM_MAX_HIGHPOWER
  91 #define CAM_MAX_HIGHPOWER  4
  92 #endif
  93
  94 /* Datastructures internal to the xpt layer */
  95 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
  96 MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
  97 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
  98 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
  99
 100 /* Object for defering XPT actions to a taskqueue */
 101 struct xpt_task {
 102         struct task     task;
 103         void            *data1;
 104         uintptr_t       data2;
 105 };
 106
 107 struct xpt_softc {
 108         uint32_t                xpt_generation;
 109
 110         /* number of high powered commands that can go through right now */
 111         struct mtx              xpt_highpower_lock;
 112         STAILQ_HEAD(highpowerlist, cam_ed)      highpowerq;
 113         int                     num_highpower;
 114
 115         /* queue for handling async rescan requests. */
 116         TAILQ_HEAD(, ccb_hdr) ccb_scanq;
 117         int buses_to_config;
 118         int buses_config_done;
 119         int announce_nosbuf;
 120
 121         /*
 122          * Registered buses
 123          *
 124          * N.B., "busses" is an archaic spelling of "buses".  In new code
 125          * "buses" is preferred.
 126          */
 127         TAILQ_HEAD(,cam_eb)     xpt_busses;
 128         u_int                   bus_generation;
 129
 130         struct intr_config_hook *xpt_config_hook;
 131
 132         int                     boot_delay;
 133         struct callout          boot_callout;
 134
 135         struct mtx              xpt_topo_lock;
 136         struct mtx              xpt_lock;
 137         struct taskqueue        *xpt_taskq;
 138 };
 139
 140 typedef enum {
 141         DM_RET_COPY             = 0x01,
 142         DM_RET_FLAG_MASK        = 0x0f,
 143         DM_RET_NONE             = 0x00,
 144         DM_RET_STOP             = 0x10,
 145         DM_RET_DESCEND          = 0x20,
 146         DM_RET_ERROR            = 0x30,
 147         DM_RET_ACTION_MASK      = 0xf0
 148 } dev_match_ret;
 149
 150 typedef enum {
 151         XPT_DEPTH_BUS,
 152         XPT_DEPTH_TARGET,
 153         XPT_DEPTH_DEVICE,
 154         XPT_DEPTH_PERIPH
 155 } xpt_traverse_depth;
 156
 157 struct xpt_traverse_config {
 158         xpt_traverse_depth      depth;
 159         void                    *tr_func;
 160         void                    *tr_arg;
 161 };
 162
 163 typedef int     xpt_busfunc_t (struct cam_eb *bus, void *arg);
 164 typedef int     xpt_targetfunc_t (struct cam_et *target, void *arg);
 165 typedef int     xpt_devicefunc_t (struct cam_ed *device, void *arg);
 166 typedef int     xpt_periphfunc_t (struct cam_periph *periph, void *arg);
 167 typedef int     xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
 168
 169 /* Transport layer configuration information */
 170 static struct xpt_softc xsoftc;
 171
 172 MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
 173
 174 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
 175            &xsoftc.boot_delay, 0, "Bus registration wait time");
 176 SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
 177             &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
 178 SYSCTL_INT(_kern_cam, OID_AUTO, announce_nosbuf, CTLFLAG_RWTUN,
 179             &xsoftc.announce_nosbuf, 0, "Don't use sbuf for announcements");
 180
 181 struct cam_doneq {
 182         struct mtx_padalign     cam_doneq_mtx;
 183         STAILQ_HEAD(, ccb_hdr)  cam_doneq;
 184         int                     cam_doneq_sleep;
 185 };
 186
 187 static struct cam_doneq cam_doneqs[MAXCPU];
 188 static int cam_num_doneqs;
 189 static struct proc *cam_proc;
 190
 191 SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
 192            &cam_num_doneqs, 0, "Number of completion queues/threads");
 193
 194 struct cam_periph *xpt_periph;
 195
 196 static periph_init_t xpt_periph_init;
 197
 198 static struct periph_driver xpt_driver =
 199 {
 200         xpt_periph_init, "xpt",
 201         TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
 202         CAM_PERIPH_DRV_EARLY
 203 };
 204
 205 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
 206
 207 static d_open_t xptopen;
 208 static d_close_t xptclose;
 209 static d_ioctl_t xptioctl;
 210 static d_ioctl_t xptdoioctl;
 211
 212 static struct cdevsw xpt_cdevsw = {
 213         .d_version =    D_VERSION,
 214         .d_flags =      0,
 215         .d_open =       xptopen,
 216         .d_close =      xptclose,
 217         .d_ioctl =      xptioctl,
 218         .d_name =       "xpt",
 219 };
 220
 221 /* Storage for debugging datastructures */
 222 struct cam_path *cam_dpath;
 223 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
 224 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RWTUN,
 225         &cam_dflags, 0, "Enabled debug flags");
 226 u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
 227 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RWTUN,
 228         &cam_debug_delay, 0, "Delay in us after each debug message");
 229
 230 /* Our boot-time initialization hook */
 231 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 232
 233 static moduledata_t cam_moduledata = {
 234         "cam",
 235         cam_module_event_handler,
 236         NULL
 237 };
 238
 239 static int      xpt_init(void *);
 240
 241 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
 242 MODULE_VERSION(cam, 1);
 243
 244
 245 static void             xpt_async_bcast(struct async_list *async_head,
 246                                         u_int32_t async_code,
 247                                         struct cam_path *path,
 248                                         void *async_arg);
 249 static path_id_t xptnextfreepathid(void);
 250 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
 251 static union ccb *xpt_get_ccb(struct cam_periph *periph);
 252 static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
 253 static void      xpt_run_allocq(struct cam_periph *periph, int sleep);
 254 static void      xpt_run_allocq_task(void *context, int pending);
 255 static void      xpt_run_devq(struct cam_devq *devq);
 256 static timeout_t xpt_release_devq_timeout;
 257 static void      xpt_release_simq_timeout(void *arg) __unused;
 258 static void      xpt_acquire_bus(struct cam_eb *bus);
 259 static void      xpt_release_bus(struct cam_eb *bus);
 260 static uint32_t  xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
 261 static int       xpt_release_devq_device(struct cam_ed *dev, u_int count,
 262                     int run_queue);
 263 static struct cam_et*
 264                  xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
 265 static void      xpt_acquire_target(struct cam_et *target);
 266 static void      xpt_release_target(struct cam_et *target);
 267 static struct cam_eb*
 268                  xpt_find_bus(path_id_t path_id);
 269 static struct cam_et*
 270                  xpt_find_target(struct cam_eb *bus, target_id_t target_id);
 271 static struct cam_ed*
 272                  xpt_find_device(struct cam_et *target, lun_id_t lun_id);
 273 static void      xpt_config(void *arg);
 274 static int       xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 275                                  u_int32_t new_priority);
 276 static xpt_devicefunc_t xptpassannouncefunc;
 277 static void      xptaction(struct cam_sim *sim, union ccb *work_ccb);
 278 static void      xptpoll(struct cam_sim *sim);
 279 static void      camisr_runqueue(void);
 280 static void      xpt_done_process(struct ccb_hdr *ccb_h);
 281 static void      xpt_done_td(void *);
 282 static dev_match_ret    xptbusmatch(struct dev_match_pattern *patterns,
 283                                     u_int num_patterns, struct cam_eb *bus);
 284 static dev_match_ret    xptdevicematch(struct dev_match_pattern *patterns,
 285                                        u_int num_patterns,
 286                                        struct cam_ed *device);
 287 static dev_match_ret    xptperiphmatch(struct dev_match_pattern *patterns,
 288                                        u_int num_patterns,
 289                                        struct cam_periph *periph);
 290 static xpt_busfunc_t    xptedtbusfunc;
 291 static xpt_targetfunc_t xptedttargetfunc;
 292 static xpt_devicefunc_t xptedtdevicefunc;
 293 static xpt_periphfunc_t xptedtperiphfunc;
 294 static xpt_pdrvfunc_t   xptplistpdrvfunc;
 295 static xpt_periphfunc_t xptplistperiphfunc;
 296 static int              xptedtmatch(struct ccb_dev_match *cdm);
 297 static int              xptperiphlistmatch(struct ccb_dev_match *cdm);
 298 static int              xptbustraverse(struct cam_eb *start_bus,
 299                                        xpt_busfunc_t *tr_func, void *arg);
 300 static int              xpttargettraverse(struct cam_eb *bus,
 301                                           struct cam_et *start_target,
 302                                           xpt_targetfunc_t *tr_func, void *arg);
 303 static int              xptdevicetraverse(struct cam_et *target,
 304                                           struct cam_ed *start_device,
 305                                           xpt_devicefunc_t *tr_func, void *arg);
 306 static int              xptperiphtraverse(struct cam_ed *device,
 307                                           struct cam_periph *start_periph,
 308                                           xpt_periphfunc_t *tr_func, void *arg);
 309 static int              xptpdrvtraverse(struct periph_driver **start_pdrv,
 310                                         xpt_pdrvfunc_t *tr_func, void *arg);
 311 static int              xptpdperiphtraverse(struct periph_driver **pdrv,
 312                                             struct cam_periph *start_periph,
 313                                             xpt_periphfunc_t *tr_func,
 314                                             void *arg);
 315 static xpt_busfunc_t    xptdefbusfunc;
 316 static xpt_targetfunc_t xptdeftargetfunc;
 317 static xpt_devicefunc_t xptdefdevicefunc;
 318 static xpt_periphfunc_t xptdefperiphfunc;
 319 static void             xpt_finishconfig_task(void *context, int pending);
 320 static void             xpt_dev_async_default(u_int32_t async_code,
 321                                               struct cam_eb *bus,
 322                                               struct cam_et *target,
 323                                               struct cam_ed *device,
 324                                               void *async_arg);
 325 static struct cam_ed *  xpt_alloc_device_default(struct cam_eb *bus,
 326                                                  struct cam_et *target,
 327                                                  lun_id_t lun_id);
 328 static xpt_devicefunc_t xptsetasyncfunc;
 329 static xpt_busfunc_t    xptsetasyncbusfunc;
 330 static cam_status       xptregister(struct cam_periph *periph,
 331                                     void *arg);
 332 static __inline int device_is_queued(struct cam_ed *device);
 333
 334 static __inline int
 335 xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
 336 {
 337         int     retval;
 338
 339         mtx_assert(&devq->send_mtx, MA_OWNED);
 340         if ((dev->ccbq.queue.entries > 0) &&
 341             (dev->ccbq.dev_openings > 0) &&
 342             (dev->ccbq.queue.qfrozen_cnt == 0)) {
 343                 /*
 344                  * The priority of a device waiting for controller
 345                  * resources is that of the highest priority CCB
 346                  * enqueued.
 347                  */
 348                 retval =
 349                     xpt_schedule_dev(&devq->send_queue,
 350                                      &dev->devq_entry,
 351                                      CAMQ_GET_PRIO(&dev->ccbq.queue));
 352         } else {
 353                 retval = 0;
 354         }
 355         return (retval);
 356 }
 357
 358 static __inline int
 359 device_is_queued(struct cam_ed *device)
 360 {
 361         return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
 362 }
 363
 364 static void
 365 xpt_periph_init()
 366 {
 367         make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
 368 }
 369
 370 static int
 371 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 372 {
 373
 374         /*
 375          * Only allow read-write access.
 376          */
 377         if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
 378                 return(EPERM);
 379
 380         /*
 381          * We don't allow nonblocking access.
 382          */
 383         if ((flags & O_NONBLOCK) != 0) {
 384                 printf("%s: can't do nonblocking access\n", devtoname(dev));
 385                 return(ENODEV);
 386         }
 387
 388         return(0);
 389 }
 390
 391 static int
 392 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 393 {
 394
 395         return(0);
 396 }
 397
 398 /*
 399  * Don't automatically grab the xpt softc lock here even though this is going
 400  * through the xpt device.  The xpt device is really just a back door for
 401  * accessing other devices and SIMs, so the right thing to do is to grab
 402  * the appropriate SIM lock once the bus/SIM is located.
 403  */
 404 static int
 405 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 406 {
 407         int error;
 408
 409         if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
 410                 error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
 411         }
 412         return (error);
 413 }
 414
 415 static int
 416 xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 417 {
 418         int error;
 419
 420         error = 0;
 421
 422         switch(cmd) {
 423         /*
 424          * For the transport layer CAMIOCOMMAND ioctl, we really only want
 425          * to accept CCB types that don't quite make sense to send through a
 426          * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
 427          * in the CAM spec.
 428          */
 429         case CAMIOCOMMAND: {
 430                 union ccb *ccb;
 431                 union ccb *inccb;
 432                 struct cam_eb *bus;
 433
 434                 inccb = (union ccb *)addr;
 435 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
 436                 if (inccb->ccb_h.func_code == XPT_SCSI_IO)
 437                         inccb->csio.bio = NULL;
 438 #endif
 439
 440                 if (inccb->ccb_h.flags & CAM_UNLOCKED)
 441                         return (EINVAL);
 442
 443                 bus = xpt_find_bus(inccb->ccb_h.path_id);
 444                 if (bus == NULL)
 445                         return (EINVAL);
 446
 447                 switch (inccb->ccb_h.func_code) {
 448                 case XPT_SCAN_BUS:
 449                 case XPT_RESET_BUS:
 450                         if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
 451                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 452                                 xpt_release_bus(bus);
 453                                 return (EINVAL);
 454                         }
 455                         break;
 456                 case XPT_SCAN_TGT:
 457                         if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
 458                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 459                                 xpt_release_bus(bus);
 460                                 return (EINVAL);
 461                         }
 462                         break;
 463                 default:
 464                         break;
 465                 }
 466
 467                 switch(inccb->ccb_h.func_code) {
 468                 case XPT_SCAN_BUS:
 469                 case XPT_RESET_BUS:
 470                 case XPT_PATH_INQ:
 471                 case XPT_ENG_INQ:
 472                 case XPT_SCAN_LUN:
 473                 case XPT_SCAN_TGT:
 474
 475                         ccb = xpt_alloc_ccb();
 476
 477                         /*
 478                          * Create a path using the bus, target, and lun the
 479                          * user passed in.
 480                          */
 481                         if (xpt_create_path(&ccb->ccb_h.path, NULL,
 482                                             inccb->ccb_h.path_id,
 483                                             inccb->ccb_h.target_id,
 484                                             inccb->ccb_h.target_lun) !=
 485                                             CAM_REQ_CMP){
 486                                 error = EINVAL;
 487                                 xpt_free_ccb(ccb);
 488                                 break;
 489                         }
 490                         /* Ensure all of our fields are correct */
 491                         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
 492                                       inccb->ccb_h.pinfo.priority);
 493                         xpt_merge_ccb(ccb, inccb);
 494                         xpt_path_lock(ccb->ccb_h.path);
 495                         cam_periph_runccb(ccb, NULL, 0, 0, NULL);
 496                         xpt_path_unlock(ccb->ccb_h.path);
 497                         bcopy(ccb, inccb, sizeof(union ccb));
 498                         xpt_free_path(ccb->ccb_h.path);
 499                         xpt_free_ccb(ccb);
 500                         break;
 501
 502                 case XPT_DEBUG: {
 503                         union ccb ccb;
 504
 505                         /*
 506                          * This is an immediate CCB, so it's okay to
 507                          * allocate it on the stack.
 508                          */
 509
 510                         /*
 511                          * Create a path using the bus, target, and lun the
 512                          * user passed in.
 513                          */
 514                         if (xpt_create_path(&ccb.ccb_h.path, NULL,
 515                                             inccb->ccb_h.path_id,
 516                                             inccb->ccb_h.target_id,
 517                                             inccb->ccb_h.target_lun) !=
 518                                             CAM_REQ_CMP){
 519                                 error = EINVAL;
 520                                 break;
 521                         }
 522                         /* Ensure all of our fields are correct */
 523                         xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
 524                                       inccb->ccb_h.pinfo.priority);
 525                         xpt_merge_ccb(&ccb, inccb);
 526                         xpt_action(&ccb);
 527                         bcopy(&ccb, inccb, sizeof(union ccb));
 528                         xpt_free_path(ccb.ccb_h.path);
 529                         break;
 530
 531                 }
 532                 case XPT_DEV_MATCH: {
 533                         struct cam_periph_map_info mapinfo;
 534                         struct cam_path *old_path;
 535
 536                         /*
 537                          * We can't deal with physical addresses for this
 538                          * type of transaction.
 539                          */
 540                         if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
 541                             CAM_DATA_VADDR) {
 542                                 error = EINVAL;
 543                                 break;
 544                         }
 545
 546                         /*
 547                          * Save this in case the caller had it set to
 548                          * something in particular.
 549                          */
 550                         old_path = inccb->ccb_h.path;
 551
 552                         /*
 553                          * We really don't need a path for the matching
 554                          * code.  The path is needed because of the
 555                          * debugging statements in xpt_action().  They
 556                          * assume that the CCB has a valid path.
 557                          */
 558                         inccb->ccb_h.path = xpt_periph->path;
 559
 560                         bzero(&mapinfo, sizeof(mapinfo));
 561
 562                         /*
 563                          * Map the pattern and match buffers into kernel
 564                          * virtual address space.
 565                          */
 566                         error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
 567
 568                         if (error) {
 569                                 inccb->ccb_h.path = old_path;
 570                                 break;
 571                         }
 572
 573                         /*
 574                          * This is an immediate CCB, we can send it on directly.
 575                          */
 576                         xpt_action(inccb);
 577
 578                         /*
 579                          * Map the buffers back into user space.
 580                          */
 581                         cam_periph_unmapmem(inccb, &mapinfo);
 582
 583                         inccb->ccb_h.path = old_path;
 584
 585                         error = 0;
 586                         break;
 587                 }
 588                 default:
 589                         error = ENOTSUP;
 590                         break;
 591                 }
 592                 xpt_release_bus(bus);
 593                 break;
 594         }
 595         /*
 596          * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
 597          * with the periphal driver name and unit name filled in.  The other
 598          * fields don't really matter as input.  The passthrough driver name
 599          * ("pass"), and unit number are passed back in the ccb.  The current
 600          * device generation number, and the index into the device peripheral
 601          * driver list, and the status are also passed back.  Note that
 602          * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
 603          * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
 604          * (or rather should be) impossible for the device peripheral driver
 605          * list to change since we look at the whole thing in one pass, and
 606          * we do it with lock protection.
 607          *
 608          */
 609         case CAMGETPASSTHRU: {
 610                 union ccb *ccb;
 611                 struct cam_periph *periph;
 612                 struct periph_driver **p_drv;
 613                 char   *name;
 614                 u_int unit;
 615                 int base_periph_found;
 616
 617                 ccb = (union ccb *)addr;
 618                 unit = ccb->cgdl.unit_number;
 619                 name = ccb->cgdl.periph_name;
 620                 base_periph_found = 0;
 621 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
 622                 if (ccb->ccb_h.func_code == XPT_SCSI_IO)
 623                         ccb->csio.bio = NULL;
 624 #endif
 625
 626                 /*
 627                  * Sanity check -- make sure we don't get a null peripheral
 628                  * driver name.
 629                  */
 630                 if (*ccb->cgdl.periph_name == '\0') {
 631                         error = EINVAL;
 632                         break;
 633                 }
 634
 635                 /* Keep the list from changing while we traverse it */
 636                 xpt_lock_buses();
 637
 638                 /* first find our driver in the list of drivers */
 639                 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
 640                         if (strcmp((*p_drv)->driver_name, name) == 0)
 641                                 break;
 642
 643                 if (*p_drv == NULL) {
 644                         xpt_unlock_buses();
 645                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 646                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 647                         *ccb->cgdl.periph_name = '\0';
 648                         ccb->cgdl.unit_number = 0;
 649                         error = ENOENT;
 650                         break;
 651                 }
 652
 653                 /*
 654                  * Run through every peripheral instance of this driver
 655                  * and check to see whether it matches the unit passed
 656                  * in by the user.  If it does, get out of the loops and
 657                  * find the passthrough driver associated with that
 658                  * peripheral driver.
 659                  */
 660                 for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
 661                      periph = TAILQ_NEXT(periph, unit_links)) {
 662
 663                         if (periph->unit_number == unit)
 664                                 break;
 665                 }
 666                 /*
 667                  * If we found the peripheral driver that the user passed
 668                  * in, go through all of the peripheral drivers for that
 669                  * particular device and look for a passthrough driver.
 670                  */
 671                 if (periph != NULL) {
 672                         struct cam_ed *device;
 673                         int i;
 674
 675                         base_periph_found = 1;
 676                         device = periph->path->device;
 677                         for (i = 0, periph = SLIST_FIRST(&device->periphs);
 678                              periph != NULL;
 679                              periph = SLIST_NEXT(periph, periph_links), i++) {
 680                                 /*
 681                                  * Check to see whether we have a
 682                                  * passthrough device or not.
 683                                  */
 684                                 if (strcmp(periph->periph_name, "pass") == 0) {
 685                                         /*
 686                                          * Fill in the getdevlist fields.
 687                                          */
 688                                         strcpy(ccb->cgdl.periph_name,
 689                                                periph->periph_name);
 690                                         ccb->cgdl.unit_number =
 691                                                 periph->unit_number;
 692                                         if (SLIST_NEXT(periph, periph_links))
 693                                                 ccb->cgdl.status =
 694                                                         CAM_GDEVLIST_MORE_DEVS;
 695                                         else
 696                                                 ccb->cgdl.status =
 697                                                        CAM_GDEVLIST_LAST_DEVICE;
 698                                         ccb->cgdl.generation =
 699                                                 device->generation;
 700                                         ccb->cgdl.index = i;
 701                                         /*
 702                                          * Fill in some CCB header fields
 703                                          * that the user may want.
 704                                          */
 705                                         ccb->ccb_h.path_id =
 706                                                 periph->path->bus->path_id;
 707                                         ccb->ccb_h.target_id =
 708                                                 periph->path->target->target_id;
 709                                         ccb->ccb_h.target_lun =
 710                                                 periph->path->device->lun_id;
 711                                         ccb->ccb_h.status = CAM_REQ_CMP;
 712                                         break;
 713                                 }
 714                         }
 715                 }
 716
 717                 /*
 718                  * If the periph is null here, one of two things has
 719                  * happened.  The first possibility is that we couldn't
 720                  * find the unit number of the particular peripheral driver
 721                  * that the user is asking about.  e.g. the user asks for
 722                  * the passthrough driver for "da11".  We find the list of
 723                  * "da" peripherals all right, but there is no unit 11.
 724                  * The other possibility is that we went through the list
 725                  * of peripheral drivers attached to the device structure,
 726                  * but didn't find one with the name "pass".  Either way,
 727                  * we return ENOENT, since we couldn't find something.
 728                  */
 729                 if (periph == NULL) {
 730                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 731                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 732                         *ccb->cgdl.periph_name = '\0';
 733                         ccb->cgdl.unit_number = 0;
 734                         error = ENOENT;
 735                         /*
 736                          * It is unfortunate that this is even necessary,
 737                          * but there are many, many clueless users out there.
 738                          * If this is true, the user is looking for the
 739                          * passthrough driver, but doesn't have one in his
 740                          * kernel.
 741                          */
 742                         if (base_periph_found == 1) {
 743                                 printf("xptioctl: pass driver is not in the "
 744                                        "kernel\n");
 745                                 printf("xptioctl: put \"device pass\" in "
 746                                        "your kernel config file\n");
 747                         }
 748                 }
 749                 xpt_unlock_buses();
 750                 break;
 751                 }
 752         default:
 753                 error = ENOTTY;
 754                 break;
 755         }
 756
 757         return(error);
 758 }
 759
 760 static int
 761 cam_module_event_handler(module_t mod, int what, void *arg)
 762 {
 763         int error;
 764
 765         switch (what) {
 766         case MOD_LOAD:
 767                 if ((error = xpt_init(NULL)) != 0)
 768                         return (error);
 769                 break;
 770         case MOD_UNLOAD:
 771                 return EBUSY;
 772         default:
 773                 return EOPNOTSUPP;
 774         }
 775
 776         return 0;
 777 }
 778
 779 static struct xpt_proto *
 780 xpt_proto_find(cam_proto proto)
 781 {
 782         struct xpt_proto **pp;
 783
 784         SET_FOREACH(pp, cam_xpt_proto_set) {
 785                 if ((*pp)->proto == proto)
 786                         return *pp;
 787         }
 788
 789         return NULL;
 790 }
 791
 792 static void
 793 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
 794 {
 795
 796         if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
 797                 xpt_free_path(done_ccb->ccb_h.path);
 798                 xpt_free_ccb(done_ccb);
 799         } else {
 800                 done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
 801                 (*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
 802         }
 803         xpt_release_boot();
 804 }
 805
 806 /* thread to handle bus rescans */
 807 static void
 808 xpt_scanner_thread(void *dummy)
 809 {
 810         union ccb       *ccb;
 811         struct cam_path  path;
 812
 813         xpt_lock_buses();
 814         for (;;) {
 815                 if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
 816                         msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
 817                                "-", 0);
 818                 if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
 819                         TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 820                         xpt_unlock_buses();
 821
 822                         printf("xpt_scanner_thread is firing on path ");
 823                         xpt_print_path(ccb->ccb_h.path);printf("\n");
 824                         /*
 825                          * Since lock can be dropped inside and path freed
 826                          * by completion callback even before return here,
 827                          * take our own path copy for reference.
 828                          */
 829                         xpt_copy_path(&path, ccb->ccb_h.path);
 830                         xpt_path_lock(&path);
 831                         xpt_action(ccb);
 832                         xpt_path_unlock(&path);
 833                         xpt_release_path(&path);
 834
 835                         xpt_lock_buses();
 836                 }
 837         }
 838 }
 839
 840 void
 841 xpt_rescan(union ccb *ccb)
 842 {
 843         struct ccb_hdr *hdr;
 844
 845         /* Prepare request */
 846         if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
 847             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 848                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 849         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 850             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 851                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 852         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 853             ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
 854                 ccb->ccb_h.func_code = XPT_SCAN_LUN;
 855         else {
 856                 xpt_print(ccb->ccb_h.path, "illegal scan path\n");
 857                 xpt_free_path(ccb->ccb_h.path);
 858                 xpt_free_ccb(ccb);
 859                 return;
 860         }
 861         CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
 862             ("xpt_rescan: func %#x %s\n", ccb->ccb_h.func_code,
 863                 xpt_action_name(ccb->ccb_h.func_code)));
 864
 865         ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
 866         ccb->ccb_h.cbfcnp = xpt_rescan_done;
 867         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
 868         /* Don't make duplicate entries for the same paths. */
 869         xpt_lock_buses();
 870         if (ccb->ccb_h.ppriv_ptr1 == NULL) {
 871                 TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
 872                         if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
 873                                 wakeup(&xsoftc.ccb_scanq);
 874                                 xpt_unlock_buses();
 875                                 xpt_print(ccb->ccb_h.path, "rescan already queued\n");
 876                                 xpt_free_path(ccb->ccb_h.path);
 877                                 xpt_free_ccb(ccb);
 878                                 return;
 879                         }
 880                 }
 881         }
 882         TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 883         xsoftc.buses_to_config++;
 884         wakeup(&xsoftc.ccb_scanq);
 885         xpt_unlock_buses();
 886 }
 887
 888 /* Functions accessed by the peripheral drivers */
 889 static int
 890 xpt_init(void *dummy)
 891 {
 892         struct cam_sim *xpt_sim;
 893         struct cam_path *path;
 894         struct cam_devq *devq;
 895         cam_status status;
 896         int error, i;
 897
 898         TAILQ_INIT(&xsoftc.xpt_busses);
 899         TAILQ_INIT(&xsoftc.ccb_scanq);
 900         STAILQ_INIT(&xsoftc.highpowerq);
 901         xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
 902
 903         mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
 904         mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
 905         xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
 906             taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
 907
 908 #ifdef CAM_BOOT_DELAY
 909         /*
 910          * Override this value at compile time to assist our users
 911          * who don't use loader to boot a kernel.
 912          */
 913         xsoftc.boot_delay = CAM_BOOT_DELAY;
 914 #endif
 915         /*
 916          * The xpt layer is, itself, the equivalent of a SIM.
 917          * Allow 16 ccbs in the ccb pool for it.  This should
 918          * give decent parallelism when we probe buses and
 919          * perform other XPT functions.
 920          */
 921         devq = cam_simq_alloc(16);
 922         xpt_sim = cam_sim_alloc(xptaction,
 923                                 xptpoll,
 924                                 "xpt",
 925                                 /*softc*/NULL,
 926                                 /*unit*/0,
 927                                 /*mtx*/&xsoftc.xpt_lock,
 928                                 /*max_dev_transactions*/0,
 929                                 /*max_tagged_dev_transactions*/0,
 930                                 devq);
 931         if (xpt_sim == NULL)
 932                 return (ENOMEM);
 933
 934         mtx_lock(&xsoftc.xpt_lock);
 935         if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
 936                 mtx_unlock(&xsoftc.xpt_lock);
 937                 printf("xpt_init: xpt_bus_register failed with status %#x,"
 938                        " failing attach\n", status);
 939                 return (EINVAL);
 940         }
 941         mtx_unlock(&xsoftc.xpt_lock);
 942
 943         /*
 944          * Looking at the XPT from the SIM layer, the XPT is
 945          * the equivalent of a peripheral driver.  Allocate
 946          * a peripheral driver entry for us.
 947          */
 948         if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
 949                                       CAM_TARGET_WILDCARD,
 950                                       CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
 951                 printf("xpt_init: xpt_create_path failed with status %#x,"
 952                        " failing attach\n", status);
 953                 return (EINVAL);
 954         }
 955         xpt_path_lock(path);
 956         cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
 957                          path, NULL, 0, xpt_sim);
 958         xpt_path_unlock(path);
 959         xpt_free_path(path);
 960
 961         if (cam_num_doneqs < 1)
 962                 cam_num_doneqs = 1 + mp_ncpus / 6;
 963         else if (cam_num_doneqs > MAXCPU)
 964                 cam_num_doneqs = MAXCPU;
 965         for (i = 0; i < cam_num_doneqs; i++) {
 966                 mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
 967                     MTX_DEF);
 968                 STAILQ_INIT(&cam_doneqs[i].cam_doneq);
 969                 error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
 970                     &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
 971                 if (error != 0) {
 972                         cam_num_doneqs = i;
 973                         break;
 974                 }
 975         }
 976         if (cam_num_doneqs < 1) {
 977                 printf("xpt_init: Cannot init completion queues "
 978                        "- failing attach\n");
 979                 return (ENOMEM);
 980         }
 981         /*
 982          * Register a callback for when interrupts are enabled.
 983          */
 984         xsoftc.xpt_config_hook =
 985             (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
 986                                               M_CAMXPT, M_NOWAIT | M_ZERO);
 987         if (xsoftc.xpt_config_hook == NULL) {
 988                 printf("xpt_init: Cannot malloc config hook "
 989                        "- failing attach\n");
 990                 return (ENOMEM);
 991         }
 992         xsoftc.xpt_config_hook->ich_func = xpt_config;
 993         if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
 994                 free (xsoftc.xpt_config_hook, M_CAMXPT);
 995                 printf("xpt_init: config_intrhook_establish failed "
 996                        "- failing attach\n");
 997         }
 998
 999         return (0);
1000 }
1001
1002 static cam_status
1003 xptregister(struct cam_periph *periph, void *arg)
1004 {
1005         struct cam_sim *xpt_sim;
1006
1007         if (periph == NULL) {
1008                 printf("xptregister: periph was NULL!!\n");
1009                 return(CAM_REQ_CMP_ERR);
1010         }
1011
1012         xpt_sim = (struct cam_sim *)arg;
1013         xpt_sim->softc = periph;
1014         xpt_periph = periph;
1015         periph->softc = NULL;
1016
1017         return(CAM_REQ_CMP);
1018 }
1019
1020 int32_t
1021 xpt_add_periph(struct cam_periph *periph)
1022 {
1023         struct cam_ed *device;
1024         int32_t  status;
1025
1026         TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
1027         device = periph->path->device;
1028         status = CAM_REQ_CMP;
1029         if (device != NULL) {
1030                 mtx_lock(&device->target->bus->eb_mtx);
1031                 device->generation++;
1032                 SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
1033                 mtx_unlock(&device->target->bus->eb_mtx);
1034                 atomic_add_32(&xsoftc.xpt_generation, 1);
1035         }
1036
1037         return (status);
1038 }
1039
1040 void
1041 xpt_remove_periph(struct cam_periph *periph)
1042 {
1043         struct cam_ed *device;
1044
1045         device = periph->path->device;
1046         if (device != NULL) {
1047                 mtx_lock(&device->target->bus->eb_mtx);
1048                 device->generation++;
1049                 SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1050                 mtx_unlock(&device->target->bus->eb_mtx);
1051                 atomic_add_32(&xsoftc.xpt_generation, 1);
1052         }
1053 }
1054
1055
1056 void
1057 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1058 {
1059         struct  cam_path *path = periph->path;
1060         struct  xpt_proto *proto;
1061
1062         cam_periph_assert(periph, MA_OWNED);
1063         periph->flags |= CAM_PERIPH_ANNOUNCED;
1064
1065         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1066                periph->periph_name, periph->unit_number,
1067                path->bus->sim->sim_name,
1068                path->bus->sim->unit_number,
1069                path->bus->sim->bus_id,
1070                path->bus->path_id,
1071                path->target->target_id,
1072                (uintmax_t)path->device->lun_id);
1073         printf("%s%d: ", periph->periph_name, periph->unit_number);
1074         proto = xpt_proto_find(path->device->protocol);
1075         if (proto)
1076                 proto->ops->announce(path->device);
1077         else
1078                 printf("%s%d: Unknown protocol device %d\n",
1079                     periph->periph_name, periph->unit_number,
1080                     path->device->protocol);
1081         if (path->device->serial_num_len > 0) {
1082                 /* Don't wrap the screen  - print only the first 60 chars */
1083                 printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1084                        periph->unit_number, path->device->serial_num);
1085         }
1086         /* Announce transport details. */
1087         path->bus->xport->ops->announce(periph);
1088         /* Announce command queueing. */
1089         if (path->device->inq_flags & SID_CmdQue
1090          || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1091                 printf("%s%d: Command Queueing enabled\n",
1092                        periph->periph_name, periph->unit_number);
1093         }
1094         /* Announce caller's details if they've passed in. */
1095         if (announce_string != NULL)
1096                 printf("%s%d: %s\n", periph->periph_name,
1097                        periph->unit_number, announce_string);
1098 }
1099
1100 void
1101 xpt_announce_periph_sbuf(struct cam_periph *periph, struct sbuf *sb,
1102     char *announce_string)
1103 {
1104         struct  cam_path *path = periph->path;
1105         struct  xpt_proto *proto;
1106
1107         cam_periph_assert(periph, MA_OWNED);
1108         periph->flags |= CAM_PERIPH_ANNOUNCED;
1109
1110         /* Fall back to the non-sbuf method if necessary */
1111         if (xsoftc.announce_nosbuf != 0) {
1112                 xpt_announce_periph(periph, announce_string);
1113                 return;
1114         }
1115         proto = xpt_proto_find(path->device->protocol);
1116         if (((proto != NULL) && (proto->ops->announce_sbuf == NULL)) ||
1117             (path->bus->xport->ops->announce_sbuf == NULL)) {
1118                 xpt_announce_periph(periph, announce_string);
1119                 return;
1120         }
1121
1122         sbuf_printf(sb, "%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1123             periph->periph_name, periph->unit_number,
1124             path->bus->sim->sim_name,
1125             path->bus->sim->unit_number,
1126             path->bus->sim->bus_id,
1127             path->bus->path_id,
1128             path->target->target_id,
1129             (uintmax_t)path->device->lun_id);
1130         sbuf_printf(sb, "%s%d: ", periph->periph_name, periph->unit_number);
1131
1132         if (proto)
1133                 proto->ops->announce_sbuf(path->device, sb);
1134         else
1135                 sbuf_printf(sb, "%s%d: Unknown protocol device %d\n",
1136                     periph->periph_name, periph->unit_number,
1137                     path->device->protocol);
1138         if (path->device->serial_num_len > 0) {
1139                 /* Don't wrap the screen  - print only the first 60 chars */
1140                 sbuf_printf(sb, "%s%d: Serial Number %.60s\n",
1141                     periph->periph_name, periph->unit_number,
1142                     path->device->serial_num);
1143         }
1144         /* Announce transport details. */
1145         path->bus->xport->ops->announce_sbuf(periph, sb);
1146         /* Announce command queueing. */
1147         if (path->device->inq_flags & SID_CmdQue
1148          || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1149                 sbuf_printf(sb, "%s%d: Command Queueing enabled\n",
1150                     periph->periph_name, periph->unit_number);
1151         }
1152         /* Announce caller's details if they've passed in. */
1153         if (announce_string != NULL)
1154                 sbuf_printf(sb, "%s%d: %s\n", periph->periph_name,
1155                     periph->unit_number, announce_string);
1156 }
1157
1158 void
1159 xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1160 {
1161         if (quirks != 0) {
1162                 printf("%s%d: quirks=0x%b\n", periph->periph_name,
1163                     periph->unit_number, quirks, bit_string);
1164         }
1165 }
1166
1167 void
1168 xpt_announce_quirks_sbuf(struct cam_periph *periph, struct sbuf *sb,
1169                          int quirks, char *bit_string)
1170 {
1171         if (xsoftc.announce_nosbuf != 0) {
1172                 xpt_announce_quirks(periph, quirks, bit_string);
1173                 return;
1174         }
1175
1176         if (quirks != 0) {
1177                 sbuf_printf(sb, "%s%d: quirks=0x%b\n", periph->periph_name,
1178                     periph->unit_number, quirks, bit_string);
1179         }
1180 }
1181
1182 void
1183 xpt_denounce_periph(struct cam_periph *periph)
1184 {
1185         struct  cam_path *path = periph->path;
1186         struct  xpt_proto *proto;
1187
1188         cam_periph_assert(periph, MA_OWNED);
1189         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1190                periph->periph_name, periph->unit_number,
1191                path->bus->sim->sim_name,
1192                path->bus->sim->unit_number,
1193                path->bus->sim->bus_id,
1194                path->bus->path_id,
1195                path->target->target_id,
1196                (uintmax_t)path->device->lun_id);
1197         printf("%s%d: ", periph->periph_name, periph->unit_number);
1198         proto = xpt_proto_find(path->device->protocol);
1199         if (proto)
1200                 proto->ops->denounce(path->device);
1201         else
1202                 printf("%s%d: Unknown protocol device %d\n",
1203                     periph->periph_name, periph->unit_number,
1204                     path->device->protocol);
1205         if (path->device->serial_num_len > 0)
1206                 printf(" s/n %.60s", path->device->serial_num);
1207         printf(" detached\n");
1208 }
1209
1210 void
1211 xpt_denounce_periph_sbuf(struct cam_periph *periph, struct sbuf *sb)
1212 {
1213         struct cam_path *path = periph->path;
1214         struct xpt_proto *proto;
1215
1216         cam_periph_assert(periph, MA_OWNED);
1217
1218         /* Fall back to the non-sbuf method if necessary */
1219         if (xsoftc.announce_nosbuf != 0) {
1220                 xpt_denounce_periph(periph);
1221                 return;
1222         }
1223         proto = xpt_proto_find(path->device->protocol);
1224         if ((proto != NULL) && (proto->ops->denounce_sbuf == NULL)) {
1225                 xpt_denounce_periph(periph);
1226                 return;
1227         }
1228
1229         sbuf_printf(sb, "%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1230             periph->periph_name, periph->unit_number,
1231             path->bus->sim->sim_name,
1232             path->bus->sim->unit_number,
1233             path->bus->sim->bus_id,
1234             path->bus->path_id,
1235             path->target->target_id,
1236             (uintmax_t)path->device->lun_id);
1237         sbuf_printf(sb, "%s%d: ", periph->periph_name, periph->unit_number);
1238
1239         if (proto)
1240                 proto->ops->denounce_sbuf(path->device, sb);
1241         else
1242                 sbuf_printf(sb, "%s%d: Unknown protocol device %d\n",
1243                     periph->periph_name, periph->unit_number,
1244                     path->device->protocol);
1245         if (path->device->serial_num_len > 0)
1246                 sbuf_printf(sb, " s/n %.60s", path->device->serial_num);
1247         sbuf_printf(sb, " detached\n");
1248 }
1249
1250 int
1251 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1252 {
1253         int ret = -1, l, o;
1254         struct ccb_dev_advinfo cdai;
1255         struct scsi_vpd_id_descriptor *idd;
1256
1257         xpt_path_assert(path, MA_OWNED);
1258
1259         memset(&cdai, 0, sizeof(cdai));
1260         xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1261         cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1262         cdai.flags = CDAI_FLAG_NONE;
1263         cdai.bufsiz = len;
1264
1265         if (!strcmp(attr, "GEOM::ident"))
1266                 cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1267         else if (!strcmp(attr, "GEOM::physpath"))
1268                 cdai.buftype = CDAI_TYPE_PHYS_PATH;
1269         else if (strcmp(attr, "GEOM::lunid") == 0 ||
1270                  strcmp(attr, "GEOM::lunname") == 0) {
1271                 cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1272                 cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1273         } else
1274                 goto out;
1275
1276         cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1277         if (cdai.buf == NULL) {
1278                 ret = ENOMEM;
1279                 goto out;
1280         }
1281         xpt_action((union ccb *)&cdai); /* can only be synchronous */
1282         if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1283                 cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1284         if (cdai.provsiz == 0)
1285                 goto out;
1286         if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1287                 if (strcmp(attr, "GEOM::lunid") == 0) {
1288                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1289                             cdai.provsiz, scsi_devid_is_lun_naa);
1290                         if (idd == NULL)
1291                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1292                                     cdai.provsiz, scsi_devid_is_lun_eui64);
1293                         if (idd == NULL)
1294                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1295                                     cdai.provsiz, scsi_devid_is_lun_uuid);
1296                         if (idd == NULL)
1297                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1298                                     cdai.provsiz, scsi_devid_is_lun_md5);
1299                 } else
1300                         idd = NULL;
1301                 if (idd == NULL)
1302                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1303                             cdai.provsiz, scsi_devid_is_lun_t10);
1304                 if (idd == NULL)
1305                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1306                             cdai.provsiz, scsi_devid_is_lun_name);
1307                 if (idd == NULL)
1308                         goto out;
1309                 ret = 0;
1310                 if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1311                         if (idd->length < len) {
1312                                 for (l = 0; l < idd->length; l++)
1313                                         buf[l] = idd->identifier[l] ?
1314                                             idd->identifier[l] : ' ';
1315                                 buf[l] = 0;
1316                         } else
1317                                 ret = EFAULT;
1318                 } else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1319                         l = strnlen(idd->identifier, idd->length);
1320                         if (l < len) {
1321                                 bcopy(idd->identifier, buf, l);
1322                                 buf[l] = 0;
1323                         } else
1324                                 ret = EFAULT;
1325                 } else if ((idd->id_type & SVPD_ID_TYPE_MASK) == SVPD_ID_TYPE_UUID
1326                     && idd->identifier[0] == 0x10) {
1327                         if ((idd->length - 2) * 2 + 4 < len) {
1328                                 for (l = 2, o = 0; l < idd->length; l++) {
1329                                         if (l == 6 || l == 8 || l == 10 || l == 12)
1330                                             o += sprintf(buf + o, "-");
1331                                         o += sprintf(buf + o, "%02x",
1332                                             idd->identifier[l]);
1333                                 }
1334                         } else
1335                                 ret = EFAULT;
1336                 } else {
1337                         if (idd->length * 2 < len) {
1338                                 for (l = 0; l < idd->length; l++)
1339                                         sprintf(buf + l * 2, "%02x",
1340                                             idd->identifier[l]);
1341                         } else
1342                                 ret = EFAULT;
1343                 }
1344         } else {
1345                 ret = 0;
1346                 if (strlcpy(buf, cdai.buf, len) >= len)
1347                         ret = EFAULT;
1348         }
1349
1350 out:
1351         if (cdai.buf != NULL)
1352                 free(cdai.buf, M_CAMXPT);
1353         return ret;
1354 }
1355
1356 static dev_match_ret
1357 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1358             struct cam_eb *bus)
1359 {
1360         dev_match_ret retval;
1361         u_int i;
1362
1363         retval = DM_RET_NONE;
1364
1365         /*
1366          * If we aren't given something to match against, that's an error.
1367          */
1368         if (bus == NULL)
1369                 return(DM_RET_ERROR);
1370
1371         /*
1372          * If there are no match entries, then this bus matches no
1373          * matter what.
1374          */
1375         if ((patterns == NULL) || (num_patterns == 0))
1376                 return(DM_RET_DESCEND | DM_RET_COPY);
1377
1378         for (i = 0; i < num_patterns; i++) {
1379                 struct bus_match_pattern *cur_pattern;
1380
1381                 /*
1382                  * If the pattern in question isn't for a bus node, we
1383                  * aren't interested.  However, we do indicate to the
1384                  * calling routine that we should continue descending the
1385                  * tree, since the user wants to match against lower-level
1386                  * EDT elements.
1387                  */
1388                 if (patterns[i].type != DEV_MATCH_BUS) {
1389                         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1390                                 retval |= DM_RET_DESCEND;
1391                         continue;
1392                 }
1393
1394                 cur_pattern = &patterns[i].pattern.bus_pattern;
1395
1396                 /*
1397                  * If they want to match any bus node, we give them any
1398                  * device node.
1399                  */
1400                 if (cur_pattern->flags == BUS_MATCH_ANY) {
1401                         /* set the copy flag */
1402                         retval |= DM_RET_COPY;
1403
1404                         /*
1405                          * If we've already decided on an action, go ahead
1406                          * and return.
1407                          */
1408                         if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1409                                 return(retval);
1410                 }
1411
1412                 /*
1413                  * Not sure why someone would do this...
1414                  */
1415                 if (cur_pattern->flags == BUS_MATCH_NONE)
1416                         continue;
1417
1418                 if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1419                  && (cur_pattern->path_id != bus->path_id))
1420                         continue;
1421
1422                 if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1423                  && (cur_pattern->bus_id != bus->sim->bus_id))
1424                         continue;
1425
1426                 if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1427                  && (cur_pattern->unit_number != bus->sim->unit_number))
1428                         continue;
1429
1430                 if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1431                  && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1432                              DEV_IDLEN) != 0))
1433                         continue;
1434
1435                 /*
1436                  * If we get to this point, the user definitely wants
1437                  * information on this bus.  So tell the caller to copy the
1438                  * data out.
1439                  */
1440                 retval |= DM_RET_COPY;
1441
1442                 /*
1443                  * If the return action has been set to descend, then we
1444                  * know that we've already seen a non-bus matching
1445                  * expression, therefore we need to further descend the tree.
1446                  * This won't change by continuing around the loop, so we
1447                  * go ahead and return.  If we haven't seen a non-bus
1448                  * matching expression, we keep going around the loop until
1449                  * we exhaust the matching expressions.  We'll set the stop
1450                  * flag once we fall out of the loop.
1451                  */
1452                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1453                         return(retval);
1454         }
1455
1456         /*
1457          * If the return action hasn't been set to descend yet, that means
1458          * we haven't seen anything other than bus matching patterns.  So
1459          * tell the caller to stop descending the tree -- the user doesn't
1460          * want to match against lower level tree elements.
1461          */
1462         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1463                 retval |= DM_RET_STOP;
1464
1465         return(retval);
1466 }
1467
1468 static dev_match_ret
1469 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1470                struct cam_ed *device)
1471 {
1472         dev_match_ret retval;
1473         u_int i;
1474
1475         retval = DM_RET_NONE;
1476
1477         /*
1478          * If we aren't given something to match against, that's an error.
1479          */
1480         if (device == NULL)
1481                 return(DM_RET_ERROR);
1482
1483         /*
1484          * If there are no match entries, then this device matches no
1485          * matter what.
1486          */
1487         if ((patterns == NULL) || (num_patterns == 0))
1488                 return(DM_RET_DESCEND | DM_RET_COPY);
1489
1490         for (i = 0; i < num_patterns; i++) {
1491                 struct device_match_pattern *cur_pattern;
1492                 struct scsi_vpd_device_id *device_id_page;
1493
1494                 /*
1495                  * If the pattern in question isn't for a device node, we
1496                  * aren't interested.
1497                  */
1498                 if (patterns[i].type != DEV_MATCH_DEVICE) {
1499                         if ((patterns[i].type == DEV_MATCH_PERIPH)
1500                          && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1501                                 retval |= DM_RET_DESCEND;
1502                         continue;
1503                 }
1504
1505                 cur_pattern = &patterns[i].pattern.device_pattern;
1506
1507                 /* Error out if mutually exclusive options are specified. */
1508                 if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1509                  == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1510                         return(DM_RET_ERROR);
1511
1512                 /*
1513                  * If they want to match any device node, we give them any
1514                  * device node.
1515                  */
1516                 if (cur_pattern->flags == DEV_MATCH_ANY)
1517                         goto copy_dev_node;
1518
1519                 /*
1520                  * Not sure why someone would do this...
1521                  */
1522                 if (cur_pattern->flags == DEV_MATCH_NONE)
1523                         continue;
1524
1525                 if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1526                  && (cur_pattern->path_id != device->target->bus->path_id))
1527                         continue;
1528
1529                 if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1530                  && (cur_pattern->target_id != device->target->target_id))
1531                         continue;
1532
1533                 if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1534                  && (cur_pattern->target_lun != device->lun_id))
1535                         continue;
1536
1537                 if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1538                  && (cam_quirkmatch((caddr_t)&device->inq_data,
1539                                     (caddr_t)&cur_pattern->data.inq_pat,
1540                                     1, sizeof(cur_pattern->data.inq_pat),
1541                                     scsi_static_inquiry_match) == NULL))
1542                         continue;
1543
1544                 device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1545                 if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1546                  && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1547                   || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1548                                       device->device_id_len
1549                                     - SVPD_DEVICE_ID_HDR_LEN,
1550                                       cur_pattern->data.devid_pat.id,
1551                                       cur_pattern->data.devid_pat.id_len) != 0))
1552                         continue;
1553
1554 copy_dev_node:
1555                 /*
1556                  * If we get to this point, the user definitely wants
1557                  * information on this device.  So tell the caller to copy
1558                  * the data out.
1559                  */
1560                 retval |= DM_RET_COPY;
1561
1562                 /*
1563                  * If the return action has been set to descend, then we
1564                  * know that we've already seen a peripheral matching
1565                  * expression, therefore we need to further descend the tree.
1566                  * This won't change by continuing around the loop, so we
1567                  * go ahead and return.  If we haven't seen a peripheral
1568                  * matching expression, we keep going around the loop until
1569                  * we exhaust the matching expressions.  We'll set the stop
1570                  * flag once we fall out of the loop.
1571                  */
1572                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1573                         return(retval);
1574         }
1575
1576         /*
1577          * If the return action hasn't been set to descend yet, that means
1578          * we haven't seen any peripheral matching patterns.  So tell the
1579          * caller to stop descending the tree -- the user doesn't want to
1580          * match against lower level tree elements.
1581          */
1582         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1583                 retval |= DM_RET_STOP;
1584
1585         return(retval);
1586 }
1587
1588 /*
1589  * Match a single peripheral against any number of match patterns.
1590  */
1591 static dev_match_ret
1592 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1593                struct cam_periph *periph)
1594 {
1595         dev_match_ret retval;
1596         u_int i;
1597
1598         /*
1599          * If we aren't given something to match against, that's an error.
1600          */
1601         if (periph == NULL)
1602                 return(DM_RET_ERROR);
1603
1604         /*
1605          * If there are no match entries, then this peripheral matches no
1606          * matter what.
1607          */
1608         if ((patterns == NULL) || (num_patterns == 0))
1609                 return(DM_RET_STOP | DM_RET_COPY);
1610
1611         /*
1612          * There aren't any nodes below a peripheral node, so there's no
1613          * reason to descend the tree any further.
1614          */
1615         retval = DM_RET_STOP;
1616
1617         for (i = 0; i < num_patterns; i++) {
1618                 struct periph_match_pattern *cur_pattern;
1619
1620                 /*
1621                  * If the pattern in question isn't for a peripheral, we
1622                  * aren't interested.
1623                  */
1624                 if (patterns[i].type != DEV_MATCH_PERIPH)
1625                         continue;
1626
1627                 cur_pattern = &patterns[i].pattern.periph_pattern;
1628
1629                 /*
1630                  * If they want to match on anything, then we will do so.
1631                  */
1632                 if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1633                         /* set the copy flag */
1634                         retval |= DM_RET_COPY;
1635
1636                         /*
1637                          * We've already set the return action to stop,
1638                          * since there are no nodes below peripherals in
1639                          * the tree.
1640                          */
1641                         return(retval);
1642                 }
1643
1644                 /*
1645                  * Not sure why someone would do this...
1646                  */
1647                 if (cur_pattern->flags == PERIPH_MATCH_NONE)
1648                         continue;
1649
1650                 if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1651                  && (cur_pattern->path_id != periph->path->bus->path_id))
1652                         continue;
1653
1654                 /*
1655                  * For the target and lun id's, we have to make sure the
1656                  * target and lun pointers aren't NULL.  The xpt peripheral
1657                  * has a wildcard target and device.
1658                  */
1659                 if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1660                  && ((periph->path->target == NULL)
1661                  ||(cur_pattern->target_id != periph->path->target->target_id)))
1662                         continue;
1663
1664                 if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1665                  && ((periph->path->device == NULL)
1666                  || (cur_pattern->target_lun != periph->path->device->lun_id)))
1667                         continue;
1668
1669                 if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1670                  && (cur_pattern->unit_number != periph->unit_number))
1671                         continue;
1672
1673                 if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1674                  && (strncmp(cur_pattern->periph_name, periph->periph_name,
1675                              DEV_IDLEN) != 0))
1676                         continue;
1677
1678                 /*
1679                  * If we get to this point, the user definitely wants
1680                  * information on this peripheral.  So tell the caller to
1681                  * copy the data out.
1682                  */
1683                 retval |= DM_RET_COPY;
1684
1685                 /*
1686                  * The return action has already been set to stop, since
1687                  * peripherals don't have any nodes below them in the EDT.
1688                  */
1689                 return(retval);
1690         }
1691
1692         /*
1693          * If we get to this point, the peripheral that was passed in
1694          * doesn't match any of the patterns.
1695          */
1696         return(retval);
1697 }
1698
1699 static int
1700 xptedtbusfunc(struct cam_eb *bus, void *arg)
1701 {
1702         struct ccb_dev_match *cdm;
1703         struct cam_et *target;
1704         dev_match_ret retval;
1705
1706         cdm = (struct ccb_dev_match *)arg;
1707
1708         /*
1709          * If our position is for something deeper in the tree, that means
1710          * that we've already seen this node.  So, we keep going down.
1711          */
1712         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1713          && (cdm->pos.cookie.bus == bus)
1714          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1715          && (cdm->pos.cookie.target != NULL))
1716                 retval = DM_RET_DESCEND;
1717         else
1718                 retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1719
1720         /*
1721          * If we got an error, bail out of the search.
1722          */
1723         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1724                 cdm->status = CAM_DEV_MATCH_ERROR;
1725                 return(0);
1726         }
1727
1728         /*
1729          * If the copy flag is set, copy this bus out.
1730          */
1731         if (retval & DM_RET_COPY) {
1732                 int spaceleft, j;
1733
1734                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1735                         sizeof(struct dev_match_result));
1736
1737                 /*
1738                  * If we don't have enough space to put in another
1739                  * match result, save our position and tell the
1740                  * user there are more devices to check.
1741                  */
1742                 if (spaceleft < sizeof(struct dev_match_result)) {
1743                         bzero(&cdm->pos, sizeof(cdm->pos));
1744                         cdm->pos.position_type =
1745                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1746
1747                         cdm->pos.cookie.bus = bus;
1748                         cdm->pos.generations[CAM_BUS_GENERATION]=
1749                                 xsoftc.bus_generation;
1750                         cdm->status = CAM_DEV_MATCH_MORE;
1751                         return(0);
1752                 }
1753                 j = cdm->num_matches;
1754                 cdm->num_matches++;
1755                 cdm->matches[j].type = DEV_MATCH_BUS;
1756                 cdm->matches[j].result.bus_result.path_id = bus->path_id;
1757                 cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1758                 cdm->matches[j].result.bus_result.unit_number =
1759                         bus->sim->unit_number;
1760                 strncpy(cdm->matches[j].result.bus_result.dev_name,
1761                         bus->sim->sim_name, DEV_IDLEN);
1762         }
1763
1764         /*
1765          * If the user is only interested in buses, there's no
1766          * reason to descend to the next level in the tree.
1767          */
1768         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1769                 return(1);
1770
1771         /*
1772          * If there is a target generation recorded, check it to
1773          * make sure the target list hasn't changed.
1774          */
1775         mtx_lock(&bus->eb_mtx);
1776         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1777          && (cdm->pos.cookie.bus == bus)
1778          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1779          && (cdm->pos.cookie.target != NULL)) {
1780                 if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1781                     bus->generation)) {
1782                         mtx_unlock(&bus->eb_mtx);
1783                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1784                         return (0);
1785                 }
1786                 target = (struct cam_et *)cdm->pos.cookie.target;
1787                 target->refcount++;
1788         } else
1789                 target = NULL;
1790         mtx_unlock(&bus->eb_mtx);
1791
1792         return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1793 }
1794
1795 static int
1796 xptedttargetfunc(struct cam_et *target, void *arg)
1797 {
1798         struct ccb_dev_match *cdm;
1799         struct cam_eb *bus;
1800         struct cam_ed *device;
1801
1802         cdm = (struct ccb_dev_match *)arg;
1803         bus = target->bus;
1804
1805         /*
1806          * If there is a device list generation recorded, check it to
1807          * make sure the device list hasn't changed.
1808          */
1809         mtx_lock(&bus->eb_mtx);
1810         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1811          && (cdm->pos.cookie.bus == bus)
1812          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1813          && (cdm->pos.cookie.target == target)
1814          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1815          && (cdm->pos.cookie.device != NULL)) {
1816                 if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1817                     target->generation) {
1818                         mtx_unlock(&bus->eb_mtx);
1819                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1820                         return(0);
1821                 }
1822                 device = (struct cam_ed *)cdm->pos.cookie.device;
1823                 device->refcount++;
1824         } else
1825                 device = NULL;
1826         mtx_unlock(&bus->eb_mtx);
1827
1828         return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1829 }
1830
1831 static int
1832 xptedtdevicefunc(struct cam_ed *device, void *arg)
1833 {
1834         struct cam_eb *bus;
1835         struct cam_periph *periph;
1836         struct ccb_dev_match *cdm;
1837         dev_match_ret retval;
1838
1839         cdm = (struct ccb_dev_match *)arg;
1840         bus = device->target->bus;
1841
1842         /*
1843          * If our position is for something deeper in the tree, that means
1844          * that we've already seen this node.  So, we keep going down.
1845          */
1846         if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1847          && (cdm->pos.cookie.device == device)
1848          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1849          && (cdm->pos.cookie.periph != NULL))
1850                 retval = DM_RET_DESCEND;
1851         else
1852                 retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1853                                         device);
1854
1855         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1856                 cdm->status = CAM_DEV_MATCH_ERROR;
1857                 return(0);
1858         }
1859
1860         /*
1861          * If the copy flag is set, copy this device out.
1862          */
1863         if (retval & DM_RET_COPY) {
1864                 int spaceleft, j;
1865
1866                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1867                         sizeof(struct dev_match_result));
1868
1869                 /*
1870                  * If we don't have enough space to put in another
1871                  * match result, save our position and tell the
1872                  * user there are more devices to check.
1873                  */
1874                 if (spaceleft < sizeof(struct dev_match_result)) {
1875                         bzero(&cdm->pos, sizeof(cdm->pos));
1876                         cdm->pos.position_type =
1877                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1878                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1879
1880                         cdm->pos.cookie.bus = device->target->bus;
1881                         cdm->pos.generations[CAM_BUS_GENERATION]=
1882                                 xsoftc.bus_generation;
1883                         cdm->pos.cookie.target = device->target;
1884                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1885                                 device->target->bus->generation;
1886                         cdm->pos.cookie.device = device;
1887                         cdm->pos.generations[CAM_DEV_GENERATION] =
1888                                 device->target->generation;
1889                         cdm->status = CAM_DEV_MATCH_MORE;
1890                         return(0);
1891                 }
1892                 j = cdm->num_matches;
1893                 cdm->num_matches++;
1894                 cdm->matches[j].type = DEV_MATCH_DEVICE;
1895                 cdm->matches[j].result.device_result.path_id =
1896                         device->target->bus->path_id;
1897                 cdm->matches[j].result.device_result.target_id =
1898                         device->target->target_id;
1899                 cdm->matches[j].result.device_result.target_lun =
1900                         device->lun_id;
1901                 cdm->matches[j].result.device_result.protocol =
1902                         device->protocol;
1903                 bcopy(&device->inq_data,
1904                       &cdm->matches[j].result.device_result.inq_data,
1905                       sizeof(struct scsi_inquiry_data));
1906                 bcopy(&device->ident_data,
1907                       &cdm->matches[j].result.device_result.ident_data,
1908                       sizeof(struct ata_params));
1909                 bcopy(&device->mmc_ident_data,
1910                       &cdm->matches[j].result.device_result.mmc_ident_data,
1911                       sizeof(struct mmc_params));
1912
1913                 /* Let the user know whether this device is unconfigured */
1914                 if (device->flags & CAM_DEV_UNCONFIGURED)
1915                         cdm->matches[j].result.device_result.flags =
1916                                 DEV_RESULT_UNCONFIGURED;
1917                 else
1918                         cdm->matches[j].result.device_result.flags =
1919                                 DEV_RESULT_NOFLAG;
1920         }
1921
1922         /*
1923          * If the user isn't interested in peripherals, don't descend
1924          * the tree any further.
1925          */
1926         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1927                 return(1);
1928
1929         /*
1930          * If there is a peripheral list generation recorded, make sure
1931          * it hasn't changed.
1932          */
1933         xpt_lock_buses();
1934         mtx_lock(&bus->eb_mtx);
1935         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1936          && (cdm->pos.cookie.bus == bus)
1937          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1938          && (cdm->pos.cookie.target == device->target)
1939          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1940          && (cdm->pos.cookie.device == device)
1941          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1942          && (cdm->pos.cookie.periph != NULL)) {
1943                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1944                     device->generation) {
1945                         mtx_unlock(&bus->eb_mtx);
1946                         xpt_unlock_buses();
1947                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1948                         return(0);
1949                 }
1950                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1951                 periph->refcount++;
1952         } else
1953                 periph = NULL;
1954         mtx_unlock(&bus->eb_mtx);
1955         xpt_unlock_buses();
1956
1957         return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1958 }
1959
1960 static int
1961 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1962 {
1963         struct ccb_dev_match *cdm;
1964         dev_match_ret retval;
1965
1966         cdm = (struct ccb_dev_match *)arg;
1967
1968         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1969
1970         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1971                 cdm->status = CAM_DEV_MATCH_ERROR;
1972                 return(0);
1973         }
1974
1975         /*
1976          * If the copy flag is set, copy this peripheral out.
1977          */
1978         if (retval & DM_RET_COPY) {
1979                 int spaceleft, j;
1980
1981                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1982                         sizeof(struct dev_match_result));
1983
1984                 /*
1985                  * If we don't have enough space to put in another
1986                  * match result, save our position and tell the
1987                  * user there are more devices to check.
1988                  */
1989                 if (spaceleft < sizeof(struct dev_match_result)) {
1990                         bzero(&cdm->pos, sizeof(cdm->pos));
1991                         cdm->pos.position_type =
1992                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1993                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1994                                 CAM_DEV_POS_PERIPH;
1995
1996                         cdm->pos.cookie.bus = periph->path->bus;
1997                         cdm->pos.generations[CAM_BUS_GENERATION]=
1998                                 xsoftc.bus_generation;
1999                         cdm->pos.cookie.target = periph->path->target;
2000                         cdm->pos.generations[CAM_TARGET_GENERATION] =
2001                                 periph->path->bus->generation;
2002                         cdm->pos.cookie.device = periph->path->device;
2003                         cdm->pos.generations[CAM_DEV_GENERATION] =
2004                                 periph->path->target->generation;
2005                         cdm->pos.cookie.periph = periph;
2006                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
2007                                 periph->path->device->generation;
2008                         cdm->status = CAM_DEV_MATCH_MORE;
2009                         return(0);
2010                 }
2011
2012                 j = cdm->num_matches;
2013                 cdm->num_matches++;
2014                 cdm->matches[j].type = DEV_MATCH_PERIPH;
2015                 cdm->matches[j].result.periph_result.path_id =
2016                         periph->path->bus->path_id;
2017                 cdm->matches[j].result.periph_result.target_id =
2018                         periph->path->target->target_id;
2019                 cdm->matches[j].result.periph_result.target_lun =
2020                         periph->path->device->lun_id;
2021                 cdm->matches[j].result.periph_result.unit_number =
2022                         periph->unit_number;
2023                 strncpy(cdm->matches[j].result.periph_result.periph_name,
2024                         periph->periph_name, DEV_IDLEN);
2025         }
2026
2027         return(1);
2028 }
2029
2030 static int
2031 xptedtmatch(struct ccb_dev_match *cdm)
2032 {
2033         struct cam_eb *bus;
2034         int ret;
2035
2036         cdm->num_matches = 0;
2037
2038         /*
2039          * Check the bus list generation.  If it has changed, the user
2040          * needs to reset everything and start over.
2041          */
2042         xpt_lock_buses();
2043         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
2044          && (cdm->pos.cookie.bus != NULL)) {
2045                 if (cdm->pos.generations[CAM_BUS_GENERATION] !=
2046                     xsoftc.bus_generation) {
2047                         xpt_unlock_buses();
2048                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
2049                         return(0);
2050                 }
2051                 bus = (struct cam_eb *)cdm->pos.cookie.bus;
2052                 bus->refcount++;
2053         } else
2054                 bus = NULL;
2055         xpt_unlock_buses();
2056
2057         ret = xptbustraverse(bus, xptedtbusfunc, cdm);
2058
2059         /*
2060          * If we get back 0, that means that we had to stop before fully
2061          * traversing the EDT.  It also means that one of the subroutines
2062          * has set the status field to the proper value.  If we get back 1,
2063          * we've fully traversed the EDT and copied out any matching entries.
2064          */
2065         if (ret == 1)
2066                 cdm->status = CAM_DEV_MATCH_LAST;
2067
2068         return(ret);
2069 }
2070
2071 static int
2072 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
2073 {
2074         struct cam_periph *periph;
2075         struct ccb_dev_match *cdm;
2076
2077         cdm = (struct ccb_dev_match *)arg;
2078
2079         xpt_lock_buses();
2080         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2081          && (cdm->pos.cookie.pdrv == pdrv)
2082          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
2083          && (cdm->pos.cookie.periph != NULL)) {
2084                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
2085                     (*pdrv)->generation) {
2086                         xpt_unlock_buses();
2087                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
2088                         return(0);
2089                 }
2090                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
2091                 periph->refcount++;
2092         } else
2093                 periph = NULL;
2094         xpt_unlock_buses();
2095
2096         return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
2097 }
2098
2099 static int
2100 xptplistperiphfunc(struct cam_periph *periph, void *arg)
2101 {
2102         struct ccb_dev_match *cdm;
2103         dev_match_ret retval;
2104
2105         cdm = (struct ccb_dev_match *)arg;
2106
2107         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
2108
2109         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
2110                 cdm->status = CAM_DEV_MATCH_ERROR;
2111                 return(0);
2112         }
2113
2114         /*
2115          * If the copy flag is set, copy this peripheral out.
2116          */
2117         if (retval & DM_RET_COPY) {
2118                 int spaceleft, j;
2119
2120                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
2121                         sizeof(struct dev_match_result));
2122
2123                 /*
2124                  * If we don't have enough space to put in another
2125                  * match result, save our position and tell the
2126                  * user there are more devices to check.
2127                  */
2128                 if (spaceleft < sizeof(struct dev_match_result)) {
2129                         struct periph_driver **pdrv;
2130
2131                         pdrv = NULL;
2132                         bzero(&cdm->pos, sizeof(cdm->pos));
2133                         cdm->pos.position_type =
2134                                 CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
2135                                 CAM_DEV_POS_PERIPH;
2136
2137                         /*
2138                          * This may look a bit non-sensical, but it is
2139                          * actually quite logical.  There are very few
2140                          * peripheral drivers, and bloating every peripheral
2141                          * structure with a pointer back to its parent
2142                          * peripheral driver linker set entry would cost
2143                          * more in the long run than doing this quick lookup.
2144                          */
2145                         for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
2146                                 if (strcmp((*pdrv)->driver_name,
2147                                     periph->periph_name) == 0)
2148                                         break;
2149                         }
2150
2151                         if (*pdrv == NULL) {
2152                                 cdm->status = CAM_DEV_MATCH_ERROR;
2153                                 return(0);
2154                         }
2155
2156                         cdm->pos.cookie.pdrv = pdrv;
2157                         /*
2158                          * The periph generation slot does double duty, as
2159                          * does the periph pointer slot.  They are used for
2160                          * both edt and pdrv lookups and positioning.
2161                          */
2162                         cdm->pos.cookie.periph = periph;
2163                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
2164                                 (*pdrv)->generation;
2165                         cdm->status = CAM_DEV_MATCH_MORE;
2166                         return(0);
2167                 }
2168
2169                 j = cdm->num_matches;
2170                 cdm->num_matches++;
2171                 cdm->matches[j].type = DEV_MATCH_PERIPH;
2172                 cdm->matches[j].result.periph_result.path_id =
2173                         periph->path->bus->path_id;
2174
2175                 /*
2176                  * The transport layer peripheral doesn't have a target or
2177                  * lun.
2178                  */
2179                 if (periph->path->target)
2180                         cdm->matches[j].result.periph_result.target_id =
2181                                 periph->path->target->target_id;
2182                 else
2183                         cdm->matches[j].result.periph_result.target_id =
2184                                 CAM_TARGET_WILDCARD;
2185
2186                 if (periph->path->device)
2187                         cdm->matches[j].result.periph_result.target_lun =
2188                                 periph->path->device->lun_id;
2189                 else
2190                         cdm->matches[j].result.periph_result.target_lun =
2191                                 CAM_LUN_WILDCARD;
2192
2193                 cdm->matches[j].result.periph_result.unit_number =
2194                         periph->unit_number;
2195                 strncpy(cdm->matches[j].result.periph_result.periph_name,
2196                         periph->periph_name, DEV_IDLEN);
2197         }
2198
2199         return(1);
2200 }
2201
2202 static int
2203 xptperiphlistmatch(struct ccb_dev_match *cdm)
2204 {
2205         int ret;
2206
2207         cdm->num_matches = 0;
2208
2209         /*
2210          * At this point in the edt traversal function, we check the bus
2211          * list generation to make sure that no buses have been added or
2212          * removed since the user last sent a XPT_DEV_MATCH ccb through.
2213          * For the peripheral driver list traversal function, however, we
2214          * don't have to worry about new peripheral driver types coming or
2215          * going; they're in a linker set, and therefore can't change
2216          * without a recompile.
2217          */
2218
2219         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2220          && (cdm->pos.cookie.pdrv != NULL))
2221                 ret = xptpdrvtraverse(
2222                                 (struct periph_driver **)cdm->pos.cookie.pdrv,
2223                                 xptplistpdrvfunc, cdm);
2224         else
2225                 ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2226
2227         /*
2228          * If we get back 0, that means that we had to stop before fully
2229          * traversing the peripheral driver tree.  It also means that one of
2230          * the subroutines has set the status field to the proper value.  If
2231          * we get back 1, we've fully traversed the EDT and copied out any
2232          * matching entries.
2233          */
2234         if (ret == 1)
2235                 cdm->status = CAM_DEV_MATCH_LAST;
2236
2237         return(ret);
2238 }
2239
2240 static int
2241 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2242 {
2243         struct cam_eb *bus, *next_bus;
2244         int retval;
2245
2246         retval = 1;
2247         if (start_bus)
2248                 bus = start_bus;
2249         else {
2250                 xpt_lock_buses();
2251                 bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2252                 if (bus == NULL) {
2253                         xpt_unlock_buses();
2254                         return (retval);
2255                 }
2256                 bus->refcount++;
2257                 xpt_unlock_buses();
2258         }
2259         for (; bus != NULL; bus = next_bus) {
2260                 retval = tr_func(bus, arg);
2261                 if (retval == 0) {
2262                         xpt_release_bus(bus);
2263                         break;
2264                 }
2265                 xpt_lock_buses();
2266                 next_bus = TAILQ_NEXT(bus, links);
2267                 if (next_bus)
2268                         next_bus->refcount++;
2269                 xpt_unlock_buses();
2270                 xpt_release_bus(bus);
2271         }
2272         return(retval);
2273 }
2274
2275 static int
2276 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2277                   xpt_targetfunc_t *tr_func, void *arg)
2278 {
2279         struct cam_et *target, *next_target;
2280         int retval;
2281
2282         retval = 1;
2283         if (start_target)
2284                 target = start_target;
2285         else {
2286                 mtx_lock(&bus->eb_mtx);
2287                 target = TAILQ_FIRST(&bus->et_entries);
2288                 if (target == NULL) {
2289                         mtx_unlock(&bus->eb_mtx);
2290                         return (retval);
2291                 }
2292                 target->refcount++;
2293                 mtx_unlock(&bus->eb_mtx);
2294         }
2295         for (; target != NULL; target = next_target) {
2296                 retval = tr_func(target, arg);
2297                 if (retval == 0) {
2298                         xpt_release_target(target);
2299                         break;
2300                 }
2301                 mtx_lock(&bus->eb_mtx);
2302                 next_target = TAILQ_NEXT(target, links);
2303                 if (next_target)
2304                         next_target->refcount++;
2305                 mtx_unlock(&bus->eb_mtx);
2306                 xpt_release_target(target);
2307         }
2308         return(retval);
2309 }
2310
2311 static int
2312 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2313                   xpt_devicefunc_t *tr_func, void *arg)
2314 {
2315         struct cam_eb *bus;
2316         struct cam_ed *device, *next_device;
2317         int retval;
2318
2319         retval = 1;
2320         bus = target->bus;
2321         if (start_device)
2322                 device = start_device;
2323         else {
2324                 mtx_lock(&bus->eb_mtx);
2325                 device = TAILQ_FIRST(&target->ed_entries);
2326                 if (device == NULL) {
2327                         mtx_unlock(&bus->eb_mtx);
2328                         return (retval);
2329                 }
2330                 device->refcount++;
2331                 mtx_unlock(&bus->eb_mtx);
2332         }
2333         for (; device != NULL; device = next_device) {
2334                 mtx_lock(&device->device_mtx);
2335                 retval = tr_func(device, arg);
2336                 mtx_unlock(&device->device_mtx);
2337                 if (retval == 0) {
2338                         xpt_release_device(device);
2339                         break;
2340                 }
2341                 mtx_lock(&bus->eb_mtx);
2342                 next_device = TAILQ_NEXT(device, links);
2343                 if (next_device)
2344                         next_device->refcount++;
2345                 mtx_unlock(&bus->eb_mtx);
2346                 xpt_release_device(device);
2347         }
2348         return(retval);
2349 }
2350
2351 static int
2352 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2353                   xpt_periphfunc_t *tr_func, void *arg)
2354 {
2355         struct cam_eb *bus;
2356         struct cam_periph *periph, *next_periph;
2357         int retval;
2358
2359         retval = 1;
2360
2361         bus = device->target->bus;
2362         if (start_periph)
2363                 periph = start_periph;
2364         else {
2365                 xpt_lock_buses();
2366                 mtx_lock(&bus->eb_mtx);
2367                 periph = SLIST_FIRST(&device->periphs);
2368                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2369                         periph = SLIST_NEXT(periph, periph_links);
2370                 if (periph == NULL) {
2371                         mtx_unlock(&bus->eb_mtx);
2372                         xpt_unlock_buses();
2373                         return (retval);
2374                 }
2375                 periph->refcount++;
2376                 mtx_unlock(&bus->eb_mtx);
2377                 xpt_unlock_buses();
2378         }
2379         for (; periph != NULL; periph = next_periph) {
2380                 retval = tr_func(periph, arg);
2381                 if (retval == 0) {
2382                         cam_periph_release_locked(periph);
2383                         break;
2384                 }
2385                 xpt_lock_buses();
2386                 mtx_lock(&bus->eb_mtx);
2387                 next_periph = SLIST_NEXT(periph, periph_links);
2388                 while (next_periph != NULL &&
2389                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2390                         next_periph = SLIST_NEXT(next_periph, periph_links);
2391                 if (next_periph)
2392                         next_periph->refcount++;
2393                 mtx_unlock(&bus->eb_mtx);
2394                 xpt_unlock_buses();
2395                 cam_periph_release_locked(periph);
2396         }
2397         return(retval);
2398 }
2399
2400 static int
2401 xptpdrvtraverse(struct periph_driver **start_pdrv,
2402                 xpt_pdrvfunc_t *tr_func, void *arg)
2403 {
2404         struct periph_driver **pdrv;
2405         int retval;
2406
2407         retval = 1;
2408
2409         /*
2410          * We don't traverse the peripheral driver list like we do the
2411          * other lists, because it is a linker set, and therefore cannot be
2412          * changed during runtime.  If the peripheral driver list is ever
2413          * re-done to be something other than a linker set (i.e. it can
2414          * change while the system is running), the list traversal should
2415          * be modified to work like the other traversal functions.
2416          */
2417         for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2418              *pdrv != NULL; pdrv++) {
2419                 retval = tr_func(pdrv, arg);
2420
2421                 if (retval == 0)
2422                         return(retval);
2423         }
2424
2425         return(retval);
2426 }
2427
2428 static int
2429 xptpdperiphtraverse(struct periph_driver **pdrv,
2430                     struct cam_periph *start_periph,
2431                     xpt_periphfunc_t *tr_func, void *arg)
2432 {
2433         struct cam_periph *periph, *next_periph;
2434         int retval;
2435
2436         retval = 1;
2437
2438         if (start_periph)
2439                 periph = start_periph;
2440         else {
2441                 xpt_lock_buses();
2442                 periph = TAILQ_FIRST(&(*pdrv)->units);
2443                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2444                         periph = TAILQ_NEXT(periph, unit_links);
2445                 if (periph == NULL) {
2446                         xpt_unlock_buses();
2447                         return (retval);
2448                 }
2449                 periph->refcount++;
2450                 xpt_unlock_buses();
2451         }
2452         for (; periph != NULL; periph = next_periph) {
2453                 cam_periph_lock(periph);
2454                 retval = tr_func(periph, arg);
2455                 cam_periph_unlock(periph);
2456                 if (retval == 0) {
2457                         cam_periph_release(periph);
2458                         break;
2459                 }
2460                 xpt_lock_buses();
2461                 next_periph = TAILQ_NEXT(periph, unit_links);
2462                 while (next_periph != NULL &&
2463                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2464                         next_periph = TAILQ_NEXT(next_periph, unit_links);
2465                 if (next_periph)
2466                         next_periph->refcount++;
2467                 xpt_unlock_buses();
2468                 cam_periph_release(periph);
2469         }
2470         return(retval);
2471 }
2472
2473 static int
2474 xptdefbusfunc(struct cam_eb *bus, void *arg)
2475 {
2476         struct xpt_traverse_config *tr_config;
2477
2478         tr_config = (struct xpt_traverse_config *)arg;
2479
2480         if (tr_config->depth == XPT_DEPTH_BUS) {
2481                 xpt_busfunc_t *tr_func;
2482
2483                 tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2484
2485                 return(tr_func(bus, tr_config->tr_arg));
2486         } else
2487                 return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2488 }
2489
2490 static int
2491 xptdeftargetfunc(struct cam_et *target, void *arg)
2492 {
2493         struct xpt_traverse_config *tr_config;
2494
2495         tr_config = (struct xpt_traverse_config *)arg;
2496
2497         if (tr_config->depth == XPT_DEPTH_TARGET) {
2498                 xpt_targetfunc_t *tr_func;
2499
2500                 tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2501
2502                 return(tr_func(target, tr_config->tr_arg));
2503         } else
2504                 return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2505 }
2506
2507 static int
2508 xptdefdevicefunc(struct cam_ed *device, void *arg)
2509 {
2510         struct xpt_traverse_config *tr_config;
2511
2512         tr_config = (struct xpt_traverse_config *)arg;
2513
2514         if (tr_config->depth == XPT_DEPTH_DEVICE) {
2515                 xpt_devicefunc_t *tr_func;
2516
2517                 tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2518
2519                 return(tr_func(device, tr_config->tr_arg));
2520         } else
2521                 return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2522 }
2523
2524 static int
2525 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2526 {
2527         struct xpt_traverse_config *tr_config;
2528         xpt_periphfunc_t *tr_func;
2529
2530         tr_config = (struct xpt_traverse_config *)arg;
2531
2532         tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2533
2534         /*
2535          * Unlike the other default functions, we don't check for depth
2536          * here.  The peripheral driver level is the last level in the EDT,
2537          * so if we're here, we should execute the function in question.
2538          */
2539         return(tr_func(periph, tr_config->tr_arg));
2540 }
2541
2542 /*
2543  * Execute the given function for every bus in the EDT.
2544  */
2545 static int
2546 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2547 {
2548         struct xpt_traverse_config tr_config;
2549
2550         tr_config.depth = XPT_DEPTH_BUS;
2551         tr_config.tr_func = tr_func;
2552         tr_config.tr_arg = arg;
2553
2554         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2555 }
2556
2557 /*
2558  * Execute the given function for every device in the EDT.
2559  */
2560 static int
2561 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2562 {
2563         struct xpt_traverse_config tr_config;
2564
2565         tr_config.depth = XPT_DEPTH_DEVICE;
2566         tr_config.tr_func = tr_func;
2567         tr_config.tr_arg = arg;
2568
2569         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2570 }
2571
2572 static int
2573 xptsetasyncfunc(struct cam_ed *device, void *arg)
2574 {
2575         struct cam_path path;
2576         struct ccb_getdev cgd;
2577         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2578
2579         /*
2580          * Don't report unconfigured devices (Wildcard devs,
2581          * devices only for target mode, device instances
2582          * that have been invalidated but are waiting for
2583          * their last reference count to be released).
2584          */
2585         if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2586                 return (1);
2587
2588         xpt_compile_path(&path,
2589                          NULL,
2590                          device->target->bus->path_id,
2591                          device->target->target_id,
2592                          device->lun_id);
2593         xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2594         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2595         xpt_action((union ccb *)&cgd);
2596         csa->callback(csa->callback_arg,
2597                             AC_FOUND_DEVICE,
2598                             &path, &cgd);
2599         xpt_release_path(&path);
2600
2601         return(1);
2602 }
2603
2604 static int
2605 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2606 {
2607         struct cam_path path;
2608         struct ccb_pathinq cpi;
2609         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2610
2611         xpt_compile_path(&path, /*periph*/NULL,
2612                          bus->path_id,
2613                          CAM_TARGET_WILDCARD,
2614                          CAM_LUN_WILDCARD);
2615         xpt_path_lock(&path);
2616         xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2617         cpi.ccb_h.func_code = XPT_PATH_INQ;
2618         xpt_action((union ccb *)&cpi);
2619         csa->callback(csa->callback_arg,
2620                             AC_PATH_REGISTERED,
2621                             &path, &cpi);
2622         xpt_path_unlock(&path);
2623         xpt_release_path(&path);
2624
2625         return(1);
2626 }
2627
2628 void
2629 xpt_action(union ccb *start_ccb)
2630 {
2631
2632         CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE,
2633             ("xpt_action: func %#x %s\n", start_ccb->ccb_h.func_code,
2634                 xpt_action_name(start_ccb->ccb_h.func_code)));
2635
2636         start_ccb->ccb_h.status = CAM_REQ_INPROG;
2637         (*(start_ccb->ccb_h.path->bus->xport->ops->action))(start_ccb);
2638 }
2639
2640 void
2641 xpt_action_default(union ccb *start_ccb)
2642 {
2643         struct cam_path *path;
2644         struct cam_sim *sim;
2645         struct mtx *mtx;
2646
2647         path = start_ccb->ccb_h.path;
2648         CAM_DEBUG(path, CAM_DEBUG_TRACE,
2649             ("xpt_action_default: func %#x %s\n", start_ccb->ccb_h.func_code,
2650                 xpt_action_name(start_ccb->ccb_h.func_code)));
2651
2652         switch (start_ccb->ccb_h.func_code) {
2653         case XPT_SCSI_IO:
2654         {
2655                 struct cam_ed *device;
2656
2657                 /*
2658                  * For the sake of compatibility with SCSI-1
2659                  * devices that may not understand the identify
2660                  * message, we include lun information in the
2661                  * second byte of all commands.  SCSI-1 specifies
2662                  * that luns are a 3 bit value and reserves only 3
2663                  * bits for lun information in the CDB.  Later
2664                  * revisions of the SCSI spec allow for more than 8
2665                  * luns, but have deprecated lun information in the
2666                  * CDB.  So, if the lun won't fit, we must omit.
2667                  *
2668                  * Also be aware that during initial probing for devices,
2669                  * the inquiry information is unknown but initialized to 0.
2670                  * This means that this code will be exercised while probing
2671                  * devices with an ANSI revision greater than 2.
2672                  */
2673                 device = path->device;
2674                 if (device->protocol_version <= SCSI_REV_2
2675                  && start_ccb->ccb_h.target_lun < 8
2676                  && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2677
2678                         start_ccb->csio.cdb_io.cdb_bytes[1] |=
2679                             start_ccb->ccb_h.target_lun << 5;
2680                 }
2681                 start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2682         }
2683         /* FALLTHROUGH */
2684         case XPT_TARGET_IO:
2685         case XPT_CONT_TARGET_IO:
2686                 start_ccb->csio.sense_resid = 0;
2687                 start_ccb->csio.resid = 0;
2688                 /* FALLTHROUGH */
2689         case XPT_ATA_IO:
2690                 if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2691                         start_ccb->ataio.resid = 0;
2692                 /* FALLTHROUGH */
2693         case XPT_NVME_IO:
2694                 if (start_ccb->ccb_h.func_code == XPT_NVME_IO)
2695                         start_ccb->nvmeio.resid = 0;
2696                 /* FALLTHROUGH */
2697         case XPT_MMC_IO:
2698                 /* XXX just like nmve_io? */
2699         case XPT_RESET_DEV:
2700         case XPT_ENG_EXEC:
2701         case XPT_SMP_IO:
2702         {
2703                 struct cam_devq *devq;
2704
2705                 devq = path->bus->sim->devq;
2706                 mtx_lock(&devq->send_mtx);
2707                 cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2708                 if (xpt_schedule_devq(devq, path->device) != 0)
2709                         xpt_run_devq(devq);
2710                 mtx_unlock(&devq->send_mtx);
2711                 break;
2712         }
2713         case XPT_CALC_GEOMETRY:
2714                 /* Filter out garbage */
2715                 if (start_ccb->ccg.block_size == 0
2716                  || start_ccb->ccg.volume_size == 0) {
2717                         start_ccb->ccg.cylinders = 0;
2718                         start_ccb->ccg.heads = 0;
2719                         start_ccb->ccg.secs_per_track = 0;
2720                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2721                         break;
2722                 }
2723 #if defined(__sparc64__)
2724                 /*
2725                  * For sparc64, we may need adjust the geometry of large
2726                  * disks in order to fit the limitations of the 16-bit
2727                  * fields of the VTOC8 disk label.
2728                  */
2729                 if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2730                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2731                         break;
2732                 }
2733 #endif
2734                 goto call_sim;
2735         case XPT_ABORT:
2736         {
2737                 union ccb* abort_ccb;
2738
2739                 abort_ccb = start_ccb->cab.abort_ccb;
2740                 if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2741                         struct cam_ed *device;
2742                         struct cam_devq *devq;
2743
2744                         device = abort_ccb->ccb_h.path->device;
2745                         devq = device->sim->devq;
2746
2747                         mtx_lock(&devq->send_mtx);
2748                         if (abort_ccb->ccb_h.pinfo.index > 0) {
2749                                 cam_ccbq_remove_ccb(&device->ccbq, abort_ccb);
2750                                 abort_ccb->ccb_h.status =
2751                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2752                                 xpt_freeze_devq_device(device, 1);
2753                                 mtx_unlock(&devq->send_mtx);
2754                                 xpt_done(abort_ccb);
2755                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2756                                 break;
2757                         }
2758                         mtx_unlock(&devq->send_mtx);
2759
2760                         if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2761                          && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2762                                 /*
2763                                  * We've caught this ccb en route to
2764                                  * the SIM.  Flag it for abort and the
2765                                  * SIM will do so just before starting
2766                                  * real work on the CCB.
2767                                  */
2768                                 abort_ccb->ccb_h.status =
2769                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2770                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2771                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2772                                 break;
2773                         }
2774                 }
2775                 if (XPT_FC_IS_QUEUED(abort_ccb)
2776                  && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2777                         /*
2778                          * It's already completed but waiting
2779                          * for our SWI to get to it.
2780                          */
2781                         start_ccb->ccb_h.status = CAM_UA_ABORT;
2782                         break;
2783                 }
2784                 /*
2785                  * If we weren't able to take care of the abort request
2786                  * in the XPT, pass the request down to the SIM for processing.
2787                  */
2788         }
2789         /* FALLTHROUGH */
2790         case XPT_ACCEPT_TARGET_IO:
2791         case XPT_EN_LUN:
2792         case XPT_IMMED_NOTIFY:
2793         case XPT_NOTIFY_ACK:
2794         case XPT_RESET_BUS:
2795         case XPT_IMMEDIATE_NOTIFY:
2796         case XPT_NOTIFY_ACKNOWLEDGE:
2797         case XPT_GET_SIM_KNOB_OLD:
2798         case XPT_GET_SIM_KNOB:
2799         case XPT_SET_SIM_KNOB:
2800         case XPT_GET_TRAN_SETTINGS:
2801         case XPT_SET_TRAN_SETTINGS:
2802         case XPT_PATH_INQ:
2803 call_sim:
2804                 sim = path->bus->sim;
2805                 mtx = sim->mtx;
2806                 if (mtx && !mtx_owned(mtx))
2807                         mtx_lock(mtx);
2808                 else
2809                         mtx = NULL;
2810
2811                 CAM_DEBUG(path, CAM_DEBUG_TRACE,
2812                     ("Calling sim->sim_action(): func=%#x\n", start_ccb->ccb_h.func_code));
2813                 (*(sim->sim_action))(sim, start_ccb);
2814                 CAM_DEBUG(path, CAM_DEBUG_TRACE,
2815                     ("sim->sim_action returned: status=%#x\n", start_ccb->ccb_h.status));
2816                 if (mtx)
2817                         mtx_unlock(mtx);
2818                 break;
2819         case XPT_PATH_STATS:
2820                 start_ccb->cpis.last_reset = path->bus->last_reset;
2821                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2822                 break;
2823         case XPT_GDEV_TYPE:
2824         {
2825                 struct cam_ed *dev;
2826
2827                 dev = path->device;
2828                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2829                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2830                 } else {
2831                         struct ccb_getdev *cgd;
2832
2833                         cgd = &start_ccb->cgd;
2834                         cgd->protocol = dev->protocol;
2835                         cgd->inq_data = dev->inq_data;
2836                         cgd->ident_data = dev->ident_data;
2837                         cgd->inq_flags = dev->inq_flags;
2838                         cgd->nvme_data = dev->nvme_data;
2839                         cgd->nvme_cdata = dev->nvme_cdata;
2840                         cgd->ccb_h.status = CAM_REQ_CMP;
2841                         cgd->serial_num_len = dev->serial_num_len;
2842                         if ((dev->serial_num_len > 0)
2843                          && (dev->serial_num != NULL))
2844                                 bcopy(dev->serial_num, cgd->serial_num,
2845                                       dev->serial_num_len);
2846                 }
2847                 break;
2848         }
2849         case XPT_GDEV_STATS:
2850         {
2851                 struct ccb_getdevstats *cgds = &start_ccb->cgds;
2852                 struct cam_ed *dev = path->device;
2853                 struct cam_eb *bus = path->bus;
2854                 struct cam_et *tar = path->target;
2855                 struct cam_devq *devq = bus->sim->devq;
2856
2857                 mtx_lock(&devq->send_mtx);
2858                 cgds->dev_openings = dev->ccbq.dev_openings;
2859                 cgds->dev_active = dev->ccbq.dev_active;
2860                 cgds->allocated = dev->ccbq.allocated;
2861                 cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2862                 cgds->held = cgds->allocated - cgds->dev_active - cgds->queued;
2863                 cgds->last_reset = tar->last_reset;
2864                 cgds->maxtags = dev->maxtags;
2865                 cgds->mintags = dev->mintags;
2866                 if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2867                         cgds->last_reset = bus->last_reset;
2868                 mtx_unlock(&devq->send_mtx);
2869                 cgds->ccb_h.status = CAM_REQ_CMP;
2870                 break;
2871         }
2872         case XPT_GDEVLIST:
2873         {
2874                 struct cam_periph       *nperiph;
2875                 struct periph_list      *periph_head;
2876                 struct ccb_getdevlist   *cgdl;
2877                 u_int                   i;
2878                 struct cam_ed           *device;
2879                 int                     found;
2880
2881
2882                 found = 0;
2883
2884                 /*
2885                  * Don't want anyone mucking with our data.
2886                  */
2887                 device = path->device;
2888                 periph_head = &device->periphs;
2889                 cgdl = &start_ccb->cgdl;
2890
2891                 /*
2892                  * Check and see if the list has changed since the user
2893                  * last requested a list member.  If so, tell them that the
2894                  * list has changed, and therefore they need to start over
2895                  * from the beginning.
2896                  */
2897                 if ((cgdl->index != 0) &&
2898                     (cgdl->generation != device->generation)) {
2899                         cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2900                         break;
2901                 }
2902
2903                 /*
2904                  * Traverse the list of peripherals and attempt to find
2905                  * the requested peripheral.
2906                  */
2907                 for (nperiph = SLIST_FIRST(periph_head), i = 0;
2908                      (nperiph != NULL) && (i <= cgdl->index);
2909                      nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2910                         if (i == cgdl->index) {
2911                                 strncpy(cgdl->periph_name,
2912                                         nperiph->periph_name,
2913                                         DEV_IDLEN);
2914                                 cgdl->unit_number = nperiph->unit_number;
2915                                 found = 1;
2916                         }
2917                 }
2918                 if (found == 0) {
2919                         cgdl->status = CAM_GDEVLIST_ERROR;
2920                         break;
2921                 }
2922
2923                 if (nperiph == NULL)
2924                         cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2925                 else
2926                         cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2927
2928                 cgdl->index++;
2929                 cgdl->generation = device->generation;
2930
2931                 cgdl->ccb_h.status = CAM_REQ_CMP;
2932                 break;
2933         }
2934         case XPT_DEV_MATCH:
2935         {
2936                 dev_pos_type position_type;
2937                 struct ccb_dev_match *cdm;
2938
2939                 cdm = &start_ccb->cdm;
2940
2941                 /*
2942                  * There are two ways of getting at information in the EDT.
2943                  * The first way is via the primary EDT tree.  It starts
2944                  * with a list of buses, then a list of targets on a bus,
2945                  * then devices/luns on a target, and then peripherals on a
2946                  * device/lun.  The "other" way is by the peripheral driver
2947                  * lists.  The peripheral driver lists are organized by
2948                  * peripheral driver.  (obviously)  So it makes sense to
2949                  * use the peripheral driver list if the user is looking
2950                  * for something like "da1", or all "da" devices.  If the
2951                  * user is looking for something on a particular bus/target
2952                  * or lun, it's generally better to go through the EDT tree.
2953                  */
2954
2955                 if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2956                         position_type = cdm->pos.position_type;
2957                 else {
2958                         u_int i;
2959
2960                         position_type = CAM_DEV_POS_NONE;
2961
2962                         for (i = 0; i < cdm->num_patterns; i++) {
2963                                 if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2964                                  ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2965                                         position_type = CAM_DEV_POS_EDT;
2966                                         break;
2967                                 }
2968                         }
2969
2970                         if (cdm->num_patterns == 0)
2971                                 position_type = CAM_DEV_POS_EDT;
2972                         else if (position_type == CAM_DEV_POS_NONE)
2973                                 position_type = CAM_DEV_POS_PDRV;
2974                 }
2975
2976                 switch(position_type & CAM_DEV_POS_TYPEMASK) {
2977                 case CAM_DEV_POS_EDT:
2978                         xptedtmatch(cdm);
2979                         break;
2980                 case CAM_DEV_POS_PDRV:
2981                         xptperiphlistmatch(cdm);
2982                         break;
2983                 default:
2984                         cdm->status = CAM_DEV_MATCH_ERROR;
2985                         break;
2986                 }
2987
2988                 if (cdm->status == CAM_DEV_MATCH_ERROR)
2989                         start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2990                 else
2991                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2992
2993                 break;
2994         }
2995         case XPT_SASYNC_CB:
2996         {
2997                 struct ccb_setasync *csa;
2998                 struct async_node *cur_entry;
2999                 struct async_list *async_head;
3000                 u_int32_t added;
3001
3002                 csa = &start_ccb->csa;
3003                 added = csa->event_enable;
3004                 async_head = &path->device->asyncs;
3005
3006                 /*
3007                  * If there is already an entry for us, simply
3008                  * update it.
3009                  */
3010                 cur_entry = SLIST_FIRST(async_head);
3011                 while (cur_entry != NULL) {
3012                         if ((cur_entry->callback_arg == csa->callback_arg)
3013                          && (cur_entry->callback == csa->callback))
3014                                 break;
3015                         cur_entry = SLIST_NEXT(cur_entry, links);
3016                 }
3017
3018                 if (cur_entry != NULL) {
3019                         /*
3020                          * If the request has no flags set,
3021                          * remove the entry.
3022                          */
3023                         added &= ~cur_entry->event_enable;
3024                         if (csa->event_enable == 0) {
3025                                 SLIST_REMOVE(async_head, cur_entry,
3026                                              async_node, links);
3027                                 xpt_release_device(path->device);
3028                                 free(cur_entry, M_CAMXPT);
3029                         } else {
3030                                 cur_entry->event_enable = csa->event_enable;
3031                         }
3032                         csa->event_enable = added;
3033                 } else {
3034                         cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
3035                                            M_NOWAIT);
3036                         if (cur_entry == NULL) {
3037                                 csa->ccb_h.status = CAM_RESRC_UNAVAIL;
3038                                 break;
3039                         }
3040                         cur_entry->event_enable = csa->event_enable;
3041                         cur_entry->event_lock = (path->bus->sim->mtx &&
3042                             mtx_owned(path->bus->sim->mtx)) ? 1 : 0;
3043                         cur_entry->callback_arg = csa->callback_arg;
3044                         cur_entry->callback = csa->callback;
3045                         SLIST_INSERT_HEAD(async_head, cur_entry, links);
3046                         xpt_acquire_device(path->device);
3047                 }
3048                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3049                 break;
3050         }
3051         case XPT_REL_SIMQ:
3052         {
3053                 struct ccb_relsim *crs;
3054                 struct cam_ed *dev;
3055
3056                 crs = &start_ccb->crs;
3057                 dev = path->device;
3058                 if (dev == NULL) {
3059
3060                         crs->ccb_h.status = CAM_DEV_NOT_THERE;
3061                         break;
3062                 }
3063
3064                 if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
3065
3066                         /* Don't ever go below one opening */
3067                         if (crs->openings > 0) {
3068                                 xpt_dev_ccbq_resize(path, crs->openings);
3069                                 if (bootverbose) {
3070                                         xpt_print(path,
3071                                             "number of openings is now %d\n",
3072                                             crs->openings);
3073                                 }
3074                         }
3075                 }
3076
3077                 mtx_lock(&dev->sim->devq->send_mtx);
3078                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
3079
3080                         if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
3081
3082                                 /*
3083                                  * Just extend the old timeout and decrement
3084                                  * the freeze count so that a single timeout
3085                                  * is sufficient for releasing the queue.
3086                                  */
3087                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3088                                 callout_stop(&dev->callout);
3089                         } else {
3090
3091                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3092                         }
3093
3094                         callout_reset_sbt(&dev->callout,
3095                             SBT_1MS * crs->release_timeout, 0,
3096                             xpt_release_devq_timeout, dev, 0);
3097
3098                         dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
3099
3100                 }
3101
3102                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
3103
3104                         if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
3105                                 /*
3106                                  * Decrement the freeze count so that a single
3107                                  * completion is still sufficient to unfreeze
3108                                  * the queue.
3109                                  */
3110                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3111                         } else {
3112
3113                                 dev->flags |= CAM_DEV_REL_ON_COMPLETE;
3114                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3115                         }
3116                 }
3117
3118                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
3119
3120                         if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
3121                          || (dev->ccbq.dev_active == 0)) {
3122
3123                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3124                         } else {
3125
3126                                 dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
3127                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3128                         }
3129                 }
3130                 mtx_unlock(&dev->sim->devq->send_mtx);
3131
3132                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
3133                         xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
3134                 start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
3135                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3136                 break;
3137         }
3138         case XPT_DEBUG: {
3139                 struct cam_path *oldpath;
3140
3141                 /* Check that all request bits are supported. */
3142                 if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
3143                         start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
3144                         break;
3145                 }
3146
3147                 cam_dflags = CAM_DEBUG_NONE;
3148                 if (cam_dpath != NULL) {
3149                         oldpath = cam_dpath;
3150                         cam_dpath = NULL;
3151                         xpt_free_path(oldpath);
3152                 }
3153                 if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
3154                         if (xpt_create_path(&cam_dpath, NULL,
3155                                             start_ccb->ccb_h.path_id,
3156                                             start_ccb->ccb_h.target_id,
3157                                             start_ccb->ccb_h.target_lun) !=
3158                                             CAM_REQ_CMP) {
3159                                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3160                         } else {
3161                                 cam_dflags = start_ccb->cdbg.flags;
3162                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3163                                 xpt_print(cam_dpath, "debugging flags now %x\n",
3164                                     cam_dflags);
3165                         }
3166                 } else
3167                         start_ccb->ccb_h.status = CAM_REQ_CMP;
3168                 break;
3169         }
3170         case XPT_NOOP:
3171                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
3172                         xpt_freeze_devq(path, 1);
3173                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3174                 break;
3175         case XPT_REPROBE_LUN:
3176                 xpt_async(AC_INQ_CHANGED, path, NULL);
3177                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3178                 xpt_done(start_ccb);
3179                 break;
3180         default:
3181         case XPT_SDEV_TYPE:
3182         case XPT_TERM_IO:
3183         case XPT_ENG_INQ:
3184                 /* XXX Implement */
3185                 xpt_print(start_ccb->ccb_h.path,
3186                     "%s: CCB type %#x %s not supported\n", __func__,
3187                     start_ccb->ccb_h.func_code,
3188                     xpt_action_name(start_ccb->ccb_h.func_code));
3189                 start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3190                 if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3191                         xpt_done(start_ccb);
3192                 }
3193                 break;
3194         }
3195         CAM_DEBUG(path, CAM_DEBUG_TRACE,
3196             ("xpt_action_default: func= %#x %s status %#x\n",
3197                 start_ccb->ccb_h.func_code,
3198                 xpt_action_name(start_ccb->ccb_h.func_code),
3199                 start_ccb->ccb_h.status));
3200 }
3201
3202 void
3203 xpt_polled_action(union ccb *start_ccb)
3204 {
3205         u_int32_t timeout;
3206         struct    cam_sim *sim;
3207         struct    cam_devq *devq;
3208         struct    cam_ed *dev;
3209         struct mtx *mtx;
3210
3211         timeout = start_ccb->ccb_h.timeout * 10;
3212         sim = start_ccb->ccb_h.path->bus->sim;
3213         devq = sim->devq;
3214         mtx = sim->mtx;
3215         dev = start_ccb->ccb_h.path->device;
3216
3217         mtx_unlock(&dev->device_mtx);
3218
3219         /*
3220          * Steal an opening so that no other queued requests
3221          * can get it before us while we simulate interrupts.
3222          */
3223         mtx_lock(&devq->send_mtx);
3224         dev->ccbq.dev_openings--;
3225         while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3226             (--timeout > 0)) {
3227                 mtx_unlock(&devq->send_mtx);
3228                 DELAY(100);
3229                 if (mtx)
3230                         mtx_lock(mtx);
3231                 (*(sim->sim_poll))(sim);
3232                 if (mtx)
3233                         mtx_unlock(mtx);
3234                 camisr_runqueue();
3235                 mtx_lock(&devq->send_mtx);
3236         }
3237         dev->ccbq.dev_openings++;
3238         mtx_unlock(&devq->send_mtx);
3239
3240         if (timeout != 0) {
3241                 xpt_action(start_ccb);
3242                 while(--timeout > 0) {
3243                         if (mtx)
3244                                 mtx_lock(mtx);
3245                         (*(sim->sim_poll))(sim);
3246                         if (mtx)
3247                                 mtx_unlock(mtx);
3248                         camisr_runqueue();
3249                         if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3250                             != CAM_REQ_INPROG)
3251                                 break;
3252                         DELAY(100);
3253                 }
3254                 if (timeout == 0) {
3255                         /*
3256                          * XXX Is it worth adding a sim_timeout entry
3257                          * point so we can attempt recovery?  If
3258                          * this is only used for dumps, I don't think
3259                          * it is.
3260                          */
3261                         start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3262                 }
3263         } else {
3264                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3265         }
3266
3267         mtx_lock(&dev->device_mtx);
3268 }
3269
3270 /*
3271  * Schedule a peripheral driver to receive a ccb when its
3272  * target device has space for more transactions.
3273  */
3274 void
3275 xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3276 {
3277
3278         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3279         cam_periph_assert(periph, MA_OWNED);
3280         if (new_priority < periph->scheduled_priority) {
3281                 periph->scheduled_priority = new_priority;
3282                 xpt_run_allocq(periph, 0);
3283         }
3284 }
3285
3286
3287 /*
3288  * Schedule a device to run on a given queue.
3289  * If the device was inserted as a new entry on the queue,
3290  * return 1 meaning the device queue should be run. If we
3291  * were already queued, implying someone else has already
3292  * started the queue, return 0 so the caller doesn't attempt
3293  * to run the queue.
3294  */
3295 static int
3296 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3297                  u_int32_t new_priority)
3298 {
3299         int retval;
3300         u_int32_t old_priority;
3301
3302         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3303
3304         old_priority = pinfo->priority;
3305
3306         /*
3307          * Are we already queued?
3308          */
3309         if (pinfo->index != CAM_UNQUEUED_INDEX) {
3310                 /* Simply reorder based on new priority */
3311                 if (new_priority < old_priority) {
3312                         camq_change_priority(queue, pinfo->index,
3313                                              new_priority);
3314                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3315                                         ("changed priority to %d\n",
3316                                          new_priority));
3317                         retval = 1;
3318                 } else
3319                         retval = 0;
3320         } else {
3321                 /* New entry on the queue */
3322                 if (new_priority < old_priority)
3323                         pinfo->priority = new_priority;
3324
3325                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3326                                 ("Inserting onto queue\n"));
3327                 pinfo->generation = ++queue->generation;
3328                 camq_insert(queue, pinfo);
3329                 retval = 1;
3330         }
3331         return (retval);
3332 }
3333
3334 static void
3335 xpt_run_allocq_task(void *context, int pending)
3336 {
3337         struct cam_periph *periph = context;
3338
3339         cam_periph_lock(periph);
3340         periph->flags &= ~CAM_PERIPH_RUN_TASK;
3341         xpt_run_allocq(periph, 1);
3342         cam_periph_unlock(periph);
3343         cam_periph_release(periph);
3344 }
3345
3346 static void
3347 xpt_run_allocq(struct cam_periph *periph, int sleep)
3348 {
3349         struct cam_ed   *device;
3350         union ccb       *ccb;
3351         uint32_t         prio;
3352
3353         cam_periph_assert(periph, MA_OWNED);
3354         if (periph->periph_allocating)
3355                 return;
3356         periph->periph_allocating = 1;
3357         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3358         device = periph->path->device;
3359         ccb = NULL;
3360 restart:
3361         while ((prio = min(periph->scheduled_priority,
3362             periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3363             (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3364              device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3365
3366                 if (ccb == NULL &&
3367                     (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3368                         if (sleep) {
3369                                 ccb = xpt_get_ccb(periph);
3370                                 goto restart;
3371                         }
3372                         if (periph->flags & CAM_PERIPH_RUN_TASK)
3373                                 break;
3374                         cam_periph_doacquire(periph);
3375                         periph->flags |= CAM_PERIPH_RUN_TASK;
3376                         taskqueue_enqueue(xsoftc.xpt_taskq,
3377                             &periph->periph_run_task);
3378                         break;
3379                 }
3380                 xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3381                 if (prio == periph->immediate_priority) {
3382                         periph->immediate_priority = CAM_PRIORITY_NONE;
3383                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3384                                         ("waking cam_periph_getccb()\n"));
3385                         SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3386                                           periph_links.sle);
3387                         wakeup(&periph->ccb_list);
3388                 } else {
3389                         periph->scheduled_priority = CAM_PRIORITY_NONE;
3390                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3391                                         ("calling periph_start()\n"));
3392                         periph->periph_start(periph, ccb);
3393                 }
3394                 ccb = NULL;
3395         }
3396         if (ccb != NULL)
3397                 xpt_release_ccb(ccb);
3398         periph->periph_allocating = 0;
3399 }
3400
3401 static void
3402 xpt_run_devq(struct cam_devq *devq)
3403 {
3404         struct mtx *mtx;
3405
3406         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3407
3408         devq->send_queue.qfrozen_cnt++;
3409         while ((devq->send_queue.entries > 0)
3410             && (devq->send_openings > 0)
3411             && (devq->send_queue.qfrozen_cnt <= 1)) {
3412                 struct  cam_ed *device;
3413                 union ccb *work_ccb;
3414                 struct  cam_sim *sim;
3415                 struct xpt_proto *proto;
3416
3417                 device = (struct cam_ed *)camq_remove(&devq->send_queue,
3418                                                            CAMQ_HEAD);
3419                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3420                                 ("running device %p\n", device));
3421
3422                 work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3423                 if (work_ccb == NULL) {
3424                         printf("device on run queue with no ccbs???\n");
3425                         continue;
3426                 }
3427
3428                 if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3429
3430                         mtx_lock(&xsoftc.xpt_highpower_lock);
3431                         if (xsoftc.num_highpower <= 0) {
3432                                 /*
3433                                  * We got a high power command, but we
3434                                  * don't have any available slots.  Freeze
3435                                  * the device queue until we have a slot
3436                                  * available.
3437                                  */
3438                                 xpt_freeze_devq_device(device, 1);
3439                                 STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3440                                                    highpowerq_entry);
3441
3442                                 mtx_unlock(&xsoftc.xpt_highpower_lock);
3443                                 continue;
3444                         } else {
3445                                 /*
3446                                  * Consume a high power slot while
3447                                  * this ccb runs.
3448                                  */
3449                                 xsoftc.num_highpower--;
3450                         }
3451                         mtx_unlock(&xsoftc.xpt_highpower_lock);
3452                 }
3453                 cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3454                 cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3455                 devq->send_openings--;
3456                 devq->send_active++;
3457                 xpt_schedule_devq(devq, device);
3458                 mtx_unlock(&devq->send_mtx);
3459
3460                 if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3461                         /*
3462                          * The client wants to freeze the queue
3463                          * after this CCB is sent.
3464                          */
3465                         xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3466                 }
3467
3468                 /* In Target mode, the peripheral driver knows best... */
3469                 if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3470                         if ((device->inq_flags & SID_CmdQue) != 0
3471                          && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3472                                 work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3473                         else
3474                                 /*
3475                                  * Clear this in case of a retried CCB that
3476                                  * failed due to a rejected tag.
3477                                  */
3478                                 work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3479                 }
3480
3481                 KASSERT(device == work_ccb->ccb_h.path->device,
3482                     ("device (%p) / path->device (%p) mismatch",
3483                         device, work_ccb->ccb_h.path->device));
3484                 proto = xpt_proto_find(device->protocol);
3485                 if (proto && proto->ops->debug_out)
3486                         proto->ops->debug_out(work_ccb);
3487
3488                 /*
3489                  * Device queues can be shared among multiple SIM instances
3490                  * that reside on different buses.  Use the SIM from the
3491                  * queued device, rather than the one from the calling bus.
3492                  */
3493                 sim = device->sim;
3494                 mtx = sim->mtx;
3495                 if (mtx && !mtx_owned(mtx))
3496                         mtx_lock(mtx);
3497                 else
3498                         mtx = NULL;
3499                 work_ccb->ccb_h.qos.sim_data = sbinuptime(); // xxx uintprt_t too small 32bit platforms
3500                 (*(sim->sim_action))(sim, work_ccb);
3501                 if (mtx)
3502                         mtx_unlock(mtx);
3503                 mtx_lock(&devq->send_mtx);
3504         }
3505         devq->send_queue.qfrozen_cnt--;
3506 }
3507
3508 /*
3509  * This function merges stuff from the slave ccb into the master ccb, while
3510  * keeping important fields in the master ccb constant.
3511  */
3512 void
3513 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3514 {
3515
3516         /*
3517          * Pull fields that are valid for peripheral drivers to set
3518          * into the master CCB along with the CCB "payload".
3519          */
3520         master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3521         master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3522         master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3523         master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3524         bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3525               sizeof(union ccb) - sizeof(struct ccb_hdr));
3526 }
3527
3528 void
3529 xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3530                     u_int32_t priority, u_int32_t flags)
3531 {
3532
3533         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3534         ccb_h->pinfo.priority = priority;
3535         ccb_h->path = path;
3536         ccb_h->path_id = path->bus->path_id;
3537         if (path->target)
3538                 ccb_h->target_id = path->target->target_id;
3539         else
3540                 ccb_h->target_id = CAM_TARGET_WILDCARD;
3541         if (path->device) {
3542                 ccb_h->target_lun = path->device->lun_id;
3543                 ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3544         } else {
3545                 ccb_h->target_lun = CAM_TARGET_WILDCARD;
3546         }
3547         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3548         ccb_h->flags = flags;
3549         ccb_h->xflags = 0;
3550 }
3551
3552 void
3553 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3554 {
3555         xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3556 }
3557
3558 /* Path manipulation functions */
3559 cam_status
3560 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3561                 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3562 {
3563         struct     cam_path *path;
3564         cam_status status;
3565
3566         path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3567
3568         if (path == NULL) {
3569                 status = CAM_RESRC_UNAVAIL;
3570                 return(status);
3571         }
3572         status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3573         if (status != CAM_REQ_CMP) {
3574                 free(path, M_CAMPATH);
3575                 path = NULL;
3576         }
3577         *new_path_ptr = path;
3578         return (status);
3579 }
3580
3581 cam_status
3582 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3583                          struct cam_periph *periph, path_id_t path_id,
3584                          target_id_t target_id, lun_id_t lun_id)
3585 {
3586
3587         return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3588             lun_id));
3589 }
3590
3591 cam_status
3592 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3593                  path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3594 {
3595         struct       cam_eb *bus;
3596         struct       cam_et *target;
3597         struct       cam_ed *device;
3598         cam_status   status;
3599
3600         status = CAM_REQ_CMP;   /* Completed without error */
3601         target = NULL;          /* Wildcarded */
3602         device = NULL;          /* Wildcarded */
3603
3604         /*
3605          * We will potentially modify the EDT, so block interrupts
3606          * that may attempt to create cam paths.
3607          */
3608         bus = xpt_find_bus(path_id);
3609         if (bus == NULL) {
3610                 status = CAM_PATH_INVALID;
3611         } else {
3612                 xpt_lock_buses();
3613                 mtx_lock(&bus->eb_mtx);
3614                 target = xpt_find_target(bus, target_id);
3615                 if (target == NULL) {
3616                         /* Create one */
3617                         struct cam_et *new_target;
3618
3619                         new_target = xpt_alloc_target(bus, target_id);
3620                         if (new_target == NULL) {
3621                                 status = CAM_RESRC_UNAVAIL;
3622                         } else {
3623                                 target = new_target;
3624                         }
3625                 }
3626                 xpt_unlock_buses();
3627                 if (target != NULL) {
3628                         device = xpt_find_device(target, lun_id);
3629                         if (device == NULL) {
3630                                 /* Create one */
3631                                 struct cam_ed *new_device;
3632
3633                                 new_device =
3634                                     (*(bus->xport->ops->alloc_device))(bus,
3635                                                                        target,
3636                                                                        lun_id);
3637                                 if (new_device == NULL) {
3638                                         status = CAM_RESRC_UNAVAIL;
3639                                 } else {
3640                                         device = new_device;
3641                                 }
3642                         }
3643                 }
3644                 mtx_unlock(&bus->eb_mtx);
3645         }
3646
3647         /*
3648          * Only touch the user's data if we are successful.
3649          */
3650         if (status == CAM_REQ_CMP) {
3651                 new_path->periph = perph;
3652                 new_path->bus = bus;
3653                 new_path->target = target;
3654                 new_path->device = device;
3655                 CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3656         } else {
3657                 if (device != NULL)
3658                         xpt_release_device(device);
3659                 if (target != NULL)
3660                         xpt_release_target(target);
3661                 if (bus != NULL)
3662                         xpt_release_bus(bus);
3663         }
3664         return (status);
3665 }
3666
3667 cam_status
3668 xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3669 {
3670         struct     cam_path *new_path;
3671
3672         new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3673         if (new_path == NULL)
3674                 return(CAM_RESRC_UNAVAIL);
3675         xpt_copy_path(new_path, path);
3676         *new_path_ptr = new_path;
3677         return (CAM_REQ_CMP);
3678 }
3679
3680 void
3681 xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3682 {
3683
3684         *new_path = *path;
3685         if (path->bus != NULL)
3686                 xpt_acquire_bus(path->bus);
3687         if (path->target != NULL)
3688                 xpt_acquire_target(path->target);
3689         if (path->device != NULL)
3690                 xpt_acquire_device(path->device);
3691 }
3692
3693 void
3694 xpt_release_path(struct cam_path *path)
3695 {
3696         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3697         if (path->device != NULL) {
3698                 xpt_release_device(path->device);
3699                 path->device = NULL;
3700         }
3701         if (path->target != NULL) {
3702                 xpt_release_target(path->target);
3703                 path->target = NULL;
3704         }
3705         if (path->bus != NULL) {
3706                 xpt_release_bus(path->bus);
3707                 path->bus = NULL;
3708         }
3709 }
3710
3711 void
3712 xpt_free_path(struct cam_path *path)
3713 {
3714
3715         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3716         xpt_release_path(path);
3717         free(path, M_CAMPATH);
3718 }
3719
3720 void
3721 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3722     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3723 {
3724
3725         xpt_lock_buses();
3726         if (bus_ref) {
3727                 if (path->bus)
3728                         *bus_ref = path->bus->refcount;
3729                 else
3730                         *bus_ref = 0;
3731         }
3732         if (periph_ref) {
3733                 if (path->periph)
3734                         *periph_ref = path->periph->refcount;
3735                 else
3736                         *periph_ref = 0;
3737         }
3738         xpt_unlock_buses();
3739         if (target_ref) {
3740                 if (path->target)
3741                         *target_ref = path->target->refcount;
3742                 else
3743                         *target_ref = 0;
3744         }
3745         if (device_ref) {
3746                 if (path->device)
3747                         *device_ref = path->device->refcount;
3748                 else
3749                         *device_ref = 0;
3750         }
3751 }
3752
3753 /*
3754  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3755  * in path1, 2 for match with wildcards in path2.
3756  */
3757 int
3758 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3759 {
3760         int retval = 0;
3761
3762         if (path1->bus != path2->bus) {
3763                 if (path1->bus->path_id == CAM_BUS_WILDCARD)
3764                         retval = 1;
3765                 else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3766                         retval = 2;
3767                 else
3768                         return (-1);
3769         }
3770         if (path1->target != path2->target) {
3771                 if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3772                         if (retval == 0)
3773                                 retval = 1;
3774                 } else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3775                         retval = 2;
3776                 else
3777                         return (-1);
3778         }
3779         if (path1->device != path2->device) {
3780                 if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3781                         if (retval == 0)
3782                                 retval = 1;
3783                 } else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3784                         retval = 2;
3785                 else
3786                         return (-1);
3787         }
3788         return (retval);
3789 }
3790
3791 int
3792 xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3793 {
3794         int retval = 0;
3795
3796         if (path->bus != dev->target->bus) {
3797                 if (path->bus->path_id == CAM_BUS_WILDCARD)
3798                         retval = 1;
3799                 else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3800                         retval = 2;
3801                 else
3802                         return (-1);
3803         }
3804         if (path->target != dev->target) {
3805                 if (path->target->target_id == CAM_TARGET_WILDCARD) {
3806                         if (retval == 0)
3807                                 retval = 1;
3808                 } else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3809                         retval = 2;
3810                 else
3811                         return (-1);
3812         }
3813         if (path->device != dev) {
3814                 if (path->device->lun_id == CAM_LUN_WILDCARD) {
3815                         if (retval == 0)
3816                                 retval = 1;
3817                 } else if (dev->lun_id == CAM_LUN_WILDCARD)
3818                         retval = 2;
3819                 else
3820                         return (-1);
3821         }
3822         return (retval);
3823 }
3824
3825 void
3826 xpt_print_path(struct cam_path *path)
3827 {
3828         struct sbuf sb;
3829         char buffer[XPT_PRINT_LEN];
3830
3831         sbuf_new(&sb, buffer, XPT_PRINT_LEN, SBUF_FIXEDLEN);
3832         xpt_path_sbuf(path, &sb);
3833         sbuf_finish(&sb);
3834         printf("%s", sbuf_data(&sb));
3835         sbuf_delete(&sb);
3836 }
3837
3838 void
3839 xpt_print_device(struct cam_ed *device)
3840 {
3841
3842         if (device == NULL)
3843                 printf("(nopath): ");
3844         else {
3845                 printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3846                        device->sim->unit_number,
3847                        device->sim->bus_id,
3848                        device->target->target_id,
3849                        (uintmax_t)device->lun_id);
3850         }
3851 }
3852
3853 void
3854 xpt_print(struct cam_path *path, const char *fmt, ...)
3855 {
3856         va_list ap;
3857         struct sbuf sb;
3858         char buffer[XPT_PRINT_LEN];
3859
3860         sbuf_new(&sb, buffer, XPT_PRINT_LEN, SBUF_FIXEDLEN);
3861
3862         xpt_path_sbuf(path, &sb);
3863         va_start(ap, fmt);
3864         sbuf_vprintf(&sb, fmt, ap);
3865         va_end(ap);
3866
3867         sbuf_finish(&sb);
3868         printf("%s", sbuf_data(&sb));
3869         sbuf_delete(&sb);
3870 }
3871
3872 int
3873 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3874 {
3875         struct sbuf sb;
3876         int len;
3877
3878         sbuf_new(&sb, str, str_len, 0);
3879         len = xpt_path_sbuf(path, &sb);
3880         sbuf_finish(&sb);
3881         return (len);
3882 }
3883
3884 int
3885 xpt_path_sbuf(struct cam_path *path, struct sbuf *sb)
3886 {
3887
3888         if (path == NULL)
3889                 sbuf_printf(sb, "(nopath): ");
3890         else {
3891                 if (path->periph != NULL)
3892                         sbuf_printf(sb, "(%s%d:", path->periph->periph_name,
3893                                     path->periph->unit_number);
3894                 else
3895                         sbuf_printf(sb, "(noperiph:");
3896
3897                 if (path->bus != NULL)
3898                         sbuf_printf(sb, "%s%d:%d:", path->bus->sim->sim_name,
3899                                     path->bus->sim->unit_number,
3900                                     path->bus->sim->bus_id);
3901                 else
3902                         sbuf_printf(sb, "nobus:");
3903
3904                 if (path->target != NULL)
3905                         sbuf_printf(sb, "%d:", path->target->target_id);
3906                 else
3907                         sbuf_printf(sb, "X:");
3908
3909                 if (path->device != NULL)
3910                         sbuf_printf(sb, "%jx): ",
3911                             (uintmax_t)path->device->lun_id);
3912                 else
3913                         sbuf_printf(sb, "X): ");
3914         }
3915
3916         return(sbuf_len(sb));
3917 }
3918
3919 path_id_t
3920 xpt_path_path_id(struct cam_path *path)
3921 {
3922         return(path->bus->path_id);
3923 }
3924
3925 target_id_t
3926 xpt_path_target_id(struct cam_path *path)
3927 {
3928         if (path->target != NULL)
3929                 return (path->target->target_id);
3930         else
3931                 return (CAM_TARGET_WILDCARD);
3932 }
3933
3934 lun_id_t
3935 xpt_path_lun_id(struct cam_path *path)
3936 {
3937         if (path->device != NULL)
3938                 return (path->device->lun_id);
3939         else
3940                 return (CAM_LUN_WILDCARD);
3941 }
3942
3943 struct cam_sim *
3944 xpt_path_sim(struct cam_path *path)
3945 {
3946
3947         return (path->bus->sim);
3948 }
3949
3950 struct cam_periph*
3951 xpt_path_periph(struct cam_path *path)
3952 {
3953
3954         return (path->periph);
3955 }
3956
3957 /*
3958  * Release a CAM control block for the caller.  Remit the cost of the structure
3959  * to the device referenced by the path.  If the this device had no 'credits'
3960  * and peripheral drivers have registered async callbacks for this notification
3961  * call them now.
3962  */
3963 void
3964 xpt_release_ccb(union ccb *free_ccb)
3965 {
3966         struct   cam_ed *device;
3967         struct   cam_periph *periph;
3968
3969         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3970         xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3971         device = free_ccb->ccb_h.path->device;
3972         periph = free_ccb->ccb_h.path->periph;
3973
3974         xpt_free_ccb(free_ccb);
3975         periph->periph_allocated--;
3976         cam_ccbq_release_opening(&device->ccbq);
3977         xpt_run_allocq(periph, 0);
3978 }
3979
3980 /* Functions accessed by SIM drivers */
3981
3982 static struct xpt_xport_ops xport_default_ops = {
3983         .alloc_device = xpt_alloc_device_default,
3984         .action = xpt_action_default,
3985         .async = xpt_dev_async_default,
3986 };
3987 static struct xpt_xport xport_default = {
3988         .xport = XPORT_UNKNOWN,
3989         .name = "unknown",
3990         .ops = &xport_default_ops,
3991 };
3992
3993 CAM_XPT_XPORT(xport_default);
3994
3995 /*
3996  * A sim structure, listing the SIM entry points and instance
3997  * identification info is passed to xpt_bus_register to hook the SIM
3998  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3999  * for this new bus and places it in the array of buses and assigns
4000  * it a path_id.  The path_id may be influenced by "hard wiring"
4001  * information specified by the user.  Once interrupt services are
4002  * available, the bus will be probed.
4003  */
4004 int32_t
4005 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
4006 {
4007         struct cam_eb *new_bus;
4008         struct cam_eb *old_bus;
4009         struct ccb_pathinq cpi;
4010         struct cam_path *path;
4011         cam_status status;
4012
4013         sim->bus_id = bus;
4014         new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
4015                                           M_CAMXPT, M_NOWAIT|M_ZERO);
4016         if (new_bus == NULL) {
4017                 /* Couldn't satisfy request */
4018                 return (CAM_RESRC_UNAVAIL);
4019         }
4020
4021         mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
4022         TAILQ_INIT(&new_bus->et_entries);
4023         cam_sim_hold(sim);
4024         new_bus->sim = sim;
4025         timevalclear(&new_bus->last_reset);
4026         new_bus->flags = 0;
4027         new_bus->refcount = 1;  /* Held until a bus_deregister event */
4028         new_bus->generation = 0;
4029
4030         xpt_lock_buses();
4031         sim->path_id = new_bus->path_id =
4032             xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
4033         old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4034         while (old_bus != NULL
4035             && old_bus->path_id < new_bus->path_id)
4036                 old_bus = TAILQ_NEXT(old_bus, links);
4037         if (old_bus != NULL)
4038                 TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
4039         else
4040                 TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
4041         xsoftc.bus_generation++;
4042         xpt_unlock_buses();
4043
4044         /*
4045          * Set a default transport so that a PATH_INQ can be issued to
4046          * the SIM.  This will then allow for probing and attaching of
4047          * a more appropriate transport.
4048          */
4049         new_bus->xport = &xport_default;
4050
4051         status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
4052                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4053         if (status != CAM_REQ_CMP) {
4054                 xpt_release_bus(new_bus);
4055                 free(path, M_CAMXPT);
4056                 return (CAM_RESRC_UNAVAIL);
4057         }
4058
4059         xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
4060         cpi.ccb_h.func_code = XPT_PATH_INQ;
4061         xpt_action((union ccb *)&cpi);
4062
4063         if (cpi.ccb_h.status == CAM_REQ_CMP) {
4064                 struct xpt_xport **xpt;
4065
4066                 SET_FOREACH(xpt, cam_xpt_xport_set) {
4067                         if ((*xpt)->xport == cpi.transport) {
4068                                 new_bus->xport = *xpt;
4069                                 break;
4070                         }
4071                 }
4072                 if (new_bus->xport == NULL) {
4073                         xpt_print(path,
4074                             "No transport found for %d\n", cpi.transport);
4075                         xpt_release_bus(new_bus);
4076                         free(path, M_CAMXPT);
4077                         return (CAM_RESRC_UNAVAIL);
4078                 }
4079         }
4080
4081         /* Notify interested parties */
4082         if (sim->path_id != CAM_XPT_PATH_ID) {
4083
4084                 xpt_async(AC_PATH_REGISTERED, path, &cpi);
4085                 if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
4086                         union   ccb *scan_ccb;
4087
4088                         /* Initiate bus rescan. */
4089                         scan_ccb = xpt_alloc_ccb_nowait();
4090                         if (scan_ccb != NULL) {
4091                                 scan_ccb->ccb_h.path = path;
4092                                 scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
4093                                 scan_ccb->crcn.flags = 0;
4094                                 xpt_rescan(scan_ccb);
4095                         } else {
4096                                 xpt_print(path,
4097                                           "Can't allocate CCB to scan bus\n");
4098                                 xpt_free_path(path);
4099                         }
4100                 } else
4101                         xpt_free_path(path);
4102         } else
4103                 xpt_free_path(path);
4104         return (CAM_SUCCESS);
4105 }
4106
4107 int32_t
4108 xpt_bus_deregister(path_id_t pathid)
4109 {
4110         struct cam_path bus_path;
4111         cam_status status;
4112
4113         status = xpt_compile_path(&bus_path, NULL, pathid,
4114                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4115         if (status != CAM_REQ_CMP)
4116                 return (status);
4117
4118         xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
4119         xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
4120
4121         /* Release the reference count held while registered. */
4122         xpt_release_bus(bus_path.bus);
4123         xpt_release_path(&bus_path);
4124
4125         return (CAM_REQ_CMP);
4126 }
4127
4128 static path_id_t
4129 xptnextfreepathid(void)
4130 {
4131         struct cam_eb *bus;
4132         path_id_t pathid;
4133         const char *strval;
4134
4135         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4136         pathid = 0;
4137         bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4138 retry:
4139         /* Find an unoccupied pathid */
4140         while (bus != NULL && bus->path_id <= pathid) {
4141                 if (bus->path_id == pathid)
4142                         pathid++;
4143                 bus = TAILQ_NEXT(bus, links);
4144         }
4145
4146         /*
4147          * Ensure that this pathid is not reserved for
4148          * a bus that may be registered in the future.
4149          */
4150         if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4151                 ++pathid;
4152                 /* Start the search over */
4153                 goto retry;
4154         }
4155         return (pathid);
4156 }
4157
4158 static path_id_t
4159 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4160 {
4161         path_id_t pathid;
4162         int i, dunit, val;
4163         char buf[32];
4164         const char *dname;
4165
4166         pathid = CAM_XPT_PATH_ID;
4167         snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4168         if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4169                 return (pathid);
4170         i = 0;
4171         while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4172                 if (strcmp(dname, "scbus")) {
4173                         /* Avoid a bit of foot shooting. */
4174                         continue;
4175                 }
4176                 if (dunit < 0)          /* unwired?! */
4177                         continue;
4178                 if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4179                         if (sim_bus == val) {
4180                                 pathid = dunit;
4181                                 break;
4182                         }
4183                 } else if (sim_bus == 0) {
4184                         /* Unspecified matches bus 0 */
4185                         pathid = dunit;
4186                         break;
4187                 } else {
4188                         printf("Ambiguous scbus configuration for %s%d "
4189                                "bus %d, cannot wire down.  The kernel "
4190                                "config entry for scbus%d should "
4191                                "specify a controller bus.\n"
4192                                "Scbus will be assigned dynamically.\n",
4193                                sim_name, sim_unit, sim_bus, dunit);
4194                         break;
4195                 }
4196         }
4197
4198         if (pathid == CAM_XPT_PATH_ID)
4199                 pathid = xptnextfreepathid();
4200         return (pathid);
4201 }
4202
4203 static const char *
4204 xpt_async_string(u_int32_t async_code)
4205 {
4206
4207         switch (async_code) {
4208         case AC_BUS_RESET: return ("AC_BUS_RESET");
4209         case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4210         case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4211         case AC_SENT_BDR: return ("AC_SENT_BDR");
4212         case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4213         case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4214         case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4215         case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4216         case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4217         case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4218         case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4219         case AC_CONTRACT: return ("AC_CONTRACT");
4220         case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4221         case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4222         }
4223         return ("AC_UNKNOWN");
4224 }
4225
4226 static int
4227 xpt_async_size(u_int32_t async_code)
4228 {
4229
4230         switch (async_code) {
4231         case AC_BUS_RESET: return (0);
4232         case AC_UNSOL_RESEL: return (0);
4233         case AC_SCSI_AEN: return (0);
4234         case AC_SENT_BDR: return (0);
4235         case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4236         case AC_PATH_DEREGISTERED: return (0);
4237         case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4238         case AC_LOST_DEVICE: return (0);
4239         case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4240         case AC_INQ_CHANGED: return (0);
4241         case AC_GETDEV_CHANGED: return (0);
4242         case AC_CONTRACT: return (sizeof(struct ac_contract));
4243         case AC_ADVINFO_CHANGED: return (-1);
4244         case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4245         }
4246         return (0);
4247 }
4248
4249 static int
4250 xpt_async_process_dev(struct cam_ed *device, void *arg)
4251 {
4252         union ccb *ccb = arg;
4253         struct cam_path *path = ccb->ccb_h.path;
4254         void *async_arg = ccb->casync.async_arg_ptr;
4255         u_int32_t async_code = ccb->casync.async_code;
4256         int relock;
4257
4258         if (path->device != device
4259          && path->device->lun_id != CAM_LUN_WILDCARD
4260          && device->lun_id != CAM_LUN_WILDCARD)
4261                 return (1);
4262
4263         /*
4264          * The async callback could free the device.
4265          * If it is a broadcast async, it doesn't hold
4266          * device reference, so take our own reference.
4267          */
4268         xpt_acquire_device(device);
4269
4270         /*
4271          * If async for specific device is to be delivered to
4272          * the wildcard client, take the specific device lock.
4273          * XXX: We may need a way for client to specify it.
4274          */
4275         if ((device->lun_id == CAM_LUN_WILDCARD &&
4276              path->device->lun_id != CAM_LUN_WILDCARD) ||
4277             (device->target->target_id == CAM_TARGET_WILDCARD &&
4278              path->target->target_id != CAM_TARGET_WILDCARD) ||
4279             (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4280              path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4281                 mtx_unlock(&device->device_mtx);
4282                 xpt_path_lock(path);
4283                 relock = 1;
4284         } else
4285                 relock = 0;
4286
4287         (*(device->target->bus->xport->ops->async))(async_code,
4288             device->target->bus, device->target, device, async_arg);
4289         xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4290
4291         if (relock) {
4292                 xpt_path_unlock(path);
4293                 mtx_lock(&device->device_mtx);
4294         }
4295         xpt_release_device(device);
4296         return (1);
4297 }
4298
4299 static int
4300 xpt_async_process_tgt(struct cam_et *target, void *arg)
4301 {
4302         union ccb *ccb = arg;
4303         struct cam_path *path = ccb->ccb_h.path;
4304
4305         if (path->target != target
4306          && path->target->target_id != CAM_TARGET_WILDCARD
4307          && target->target_id != CAM_TARGET_WILDCARD)
4308                 return (1);
4309
4310         if (ccb->casync.async_code == AC_SENT_BDR) {
4311                 /* Update our notion of when the last reset occurred */
4312                 microtime(&target->last_reset);
4313         }
4314
4315         return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4316 }
4317
4318 static void
4319 xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4320 {
4321         struct cam_eb *bus;
4322         struct cam_path *path;
4323         void *async_arg;
4324         u_int32_t async_code;
4325
4326         path = ccb->ccb_h.path;
4327         async_code = ccb->casync.async_code;
4328         async_arg = ccb->casync.async_arg_ptr;
4329         CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4330             ("xpt_async(%s)\n", xpt_async_string(async_code)));
4331         bus = path->bus;
4332
4333         if (async_code == AC_BUS_RESET) {
4334                 /* Update our notion of when the last reset occurred */
4335                 microtime(&bus->last_reset);
4336         }
4337
4338         xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4339
4340         /*
4341          * If this wasn't a fully wildcarded async, tell all
4342          * clients that want all async events.
4343          */
4344         if (bus != xpt_periph->path->bus) {
4345                 xpt_path_lock(xpt_periph->path);
4346                 xpt_async_process_dev(xpt_periph->path->device, ccb);
4347                 xpt_path_unlock(xpt_periph->path);
4348         }
4349
4350         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4351                 xpt_release_devq(path, 1, TRUE);
4352         else
4353                 xpt_release_simq(path->bus->sim, TRUE);
4354         if (ccb->casync.async_arg_size > 0)
4355                 free(async_arg, M_CAMXPT);
4356         xpt_free_path(path);
4357         xpt_free_ccb(ccb);
4358 }
4359
4360 static void
4361 xpt_async_bcast(struct async_list *async_head,
4362                 u_int32_t async_code,
4363                 struct cam_path *path, void *async_arg)
4364 {
4365         struct async_node *cur_entry;
4366         struct mtx *mtx;
4367
4368         cur_entry = SLIST_FIRST(async_head);
4369         while (cur_entry != NULL) {
4370                 struct async_node *next_entry;
4371                 /*
4372                  * Grab the next list entry before we call the current
4373                  * entry's callback.  This is because the callback function
4374                  * can delete its async callback entry.
4375                  */
4376                 next_entry = SLIST_NEXT(cur_entry, links);
4377                 if ((cur_entry->event_enable & async_code) != 0) {
4378                         mtx = cur_entry->event_lock ?
4379                             path->device->sim->mtx : NULL;
4380                         if (mtx)
4381                                 mtx_lock(mtx);
4382                         cur_entry->callback(cur_entry->callback_arg,
4383                                             async_code, path,
4384                                             async_arg);
4385                         if (mtx)
4386                                 mtx_unlock(mtx);
4387                 }
4388                 cur_entry = next_entry;
4389         }
4390 }
4391
4392 void
4393 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4394 {
4395         union ccb *ccb;
4396         int size;
4397
4398         ccb = xpt_alloc_ccb_nowait();
4399         if (ccb == NULL) {
4400                 xpt_print(path, "Can't allocate CCB to send %s\n",
4401                     xpt_async_string(async_code));
4402                 return;
4403         }
4404
4405         if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4406                 xpt_print(path, "Can't allocate path to send %s\n",
4407                     xpt_async_string(async_code));
4408                 xpt_free_ccb(ccb);
4409                 return;
4410         }
4411         ccb->ccb_h.path->periph = NULL;
4412         ccb->ccb_h.func_code = XPT_ASYNC;
4413         ccb->ccb_h.cbfcnp = xpt_async_process;
4414         ccb->ccb_h.flags |= CAM_UNLOCKED;
4415         ccb->casync.async_code = async_code;
4416         ccb->casync.async_arg_size = 0;
4417         size = xpt_async_size(async_code);
4418         CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
4419             ("xpt_async: func %#x %s aync_code %d %s\n",
4420                 ccb->ccb_h.func_code,
4421                 xpt_action_name(ccb->ccb_h.func_code),
4422                 async_code,
4423                 xpt_async_string(async_code)));
4424         if (size > 0 && async_arg != NULL) {
4425                 ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4426                 if (ccb->casync.async_arg_ptr == NULL) {
4427                         xpt_print(path, "Can't allocate argument to send %s\n",
4428                             xpt_async_string(async_code));
4429                         xpt_free_path(ccb->ccb_h.path);
4430                         xpt_free_ccb(ccb);
4431                         return;
4432                 }
4433                 memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4434                 ccb->casync.async_arg_size = size;
4435         } else if (size < 0) {
4436                 ccb->casync.async_arg_ptr = async_arg;
4437                 ccb->casync.async_arg_size = size;
4438         }
4439         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4440                 xpt_freeze_devq(path, 1);
4441         else
4442                 xpt_freeze_simq(path->bus->sim, 1);
4443         xpt_done(ccb);
4444 }
4445
4446 static void
4447 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4448                       struct cam_et *target, struct cam_ed *device,
4449                       void *async_arg)
4450 {
4451
4452         /*
4453          * We only need to handle events for real devices.
4454          */
4455         if (target->target_id == CAM_TARGET_WILDCARD
4456          || device->lun_id == CAM_LUN_WILDCARD)
4457                 return;
4458
4459         printf("%s called\n", __func__);
4460 }
4461
4462 static uint32_t
4463 xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4464 {
4465         struct cam_devq *devq;
4466         uint32_t freeze;
4467
4468         devq = dev->sim->devq;
4469         mtx_assert(&devq->send_mtx, MA_OWNED);
4470         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4471             ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4472             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4473         freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4474         /* Remove frozen device from sendq. */
4475         if (device_is_queued(dev))
4476                 camq_remove(&devq->send_queue, dev->devq_entry.index);
4477         return (freeze);
4478 }
4479
4480 u_int32_t
4481 xpt_freeze_devq(struct cam_path *path, u_int count)
4482 {
4483         struct cam_ed   *dev = path->device;
4484         struct cam_devq *devq;
4485         uint32_t         freeze;
4486
4487         devq = dev->sim->devq;
4488         mtx_lock(&devq->send_mtx);
4489         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4490         freeze = xpt_freeze_devq_device(dev, count);
4491         mtx_unlock(&devq->send_mtx);
4492         return (freeze);
4493 }
4494
4495 u_int32_t
4496 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4497 {
4498         struct cam_devq *devq;
4499         uint32_t         freeze;
4500
4501         devq = sim->devq;
4502         mtx_lock(&devq->send_mtx);
4503         freeze = (devq->send_queue.qfrozen_cnt += count);
4504         mtx_unlock(&devq->send_mtx);
4505         return (freeze);
4506 }
4507
4508 static void
4509 xpt_release_devq_timeout(void *arg)
4510 {
4511         struct cam_ed *dev;
4512         struct cam_devq *devq;
4513
4514         dev = (struct cam_ed *)arg;
4515         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4516         devq = dev->sim->devq;
4517         mtx_assert(&devq->send_mtx, MA_OWNED);
4518         if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4519                 xpt_run_devq(devq);
4520 }
4521
4522 void
4523 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4524 {
4525         struct cam_ed *dev;
4526         struct cam_devq *devq;
4527
4528         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4529             count, run_queue));
4530         dev = path->device;
4531         devq = dev->sim->devq;
4532         mtx_lock(&devq->send_mtx);
4533         if (xpt_release_devq_device(dev, count, run_queue))
4534                 xpt_run_devq(dev->sim->devq);
4535         mtx_unlock(&devq->send_mtx);
4536 }
4537
4538 static int
4539 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4540 {
4541
4542         mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4543         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4544             ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4545             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4546         if (count > dev->ccbq.queue.qfrozen_cnt) {
4547 #ifdef INVARIANTS
4548                 printf("xpt_release_devq(): requested %u > present %u\n",
4549                     count, dev->ccbq.queue.qfrozen_cnt);
4550 #endif
4551                 count = dev->ccbq.queue.qfrozen_cnt;
4552         }
4553         dev->ccbq.queue.qfrozen_cnt -= count;
4554         if (dev->ccbq.queue.qfrozen_cnt == 0) {
4555                 /*
4556                  * No longer need to wait for a successful
4557                  * command completion.
4558                  */
4559                 dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4560                 /*
4561                  * Remove any timeouts that might be scheduled
4562                  * to release this queue.
4563                  */
4564                 if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4565                         callout_stop(&dev->callout);
4566                         dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4567                 }
4568                 /*
4569                  * Now that we are unfrozen schedule the
4570                  * device so any pending transactions are
4571                  * run.
4572                  */
4573                 xpt_schedule_devq(dev->sim->devq, dev);
4574         } else
4575                 run_queue = 0;
4576         return (run_queue);
4577 }
4578
4579 void
4580 xpt_release_simq(struct cam_sim *sim, int run_queue)
4581 {
4582         struct cam_devq *devq;
4583
4584         devq = sim->devq;
4585         mtx_lock(&devq->send_mtx);
4586         if (devq->send_queue.qfrozen_cnt <= 0) {
4587 #ifdef INVARIANTS
4588                 printf("xpt_release_simq: requested 1 > present %u\n",
4589                     devq->send_queue.qfrozen_cnt);
4590 #endif
4591         } else
4592                 devq->send_queue.qfrozen_cnt--;
4593         if (devq->send_queue.qfrozen_cnt == 0) {
4594                 /*
4595                  * If there is a timeout scheduled to release this
4596                  * sim queue, remove it.  The queue frozen count is
4597                  * already at 0.
4598                  */
4599                 if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4600                         callout_stop(&sim->callout);
4601                         sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4602                 }
4603                 if (run_queue) {
4604                         /*
4605                          * Now that we are unfrozen run the send queue.
4606                          */
4607                         xpt_run_devq(sim->devq);
4608                 }
4609         }
4610         mtx_unlock(&devq->send_mtx);
4611 }
4612
4613 /*
4614  * XXX Appears to be unused.
4615  */
4616 static void
4617 xpt_release_simq_timeout(void *arg)
4618 {
4619         struct cam_sim *sim;
4620
4621         sim = (struct cam_sim *)arg;
4622         xpt_release_simq(sim, /* run_queue */ TRUE);
4623 }
4624
4625 void
4626 xpt_done(union ccb *done_ccb)
4627 {
4628         struct cam_doneq *queue;
4629         int     run, hash;
4630
4631 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
4632         if (done_ccb->ccb_h.func_code == XPT_SCSI_IO &&
4633             done_ccb->csio.bio != NULL)
4634                 biotrack(done_ccb->csio.bio, __func__);
4635 #endif
4636
4637         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
4638             ("xpt_done: func= %#x %s status %#x\n",
4639                 done_ccb->ccb_h.func_code,
4640                 xpt_action_name(done_ccb->ccb_h.func_code),
4641                 done_ccb->ccb_h.status));
4642         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4643                 return;
4644
4645         /* Store the time the ccb was in the sim */
4646         done_ccb->ccb_h.qos.sim_data = sbinuptime() - done_ccb->ccb_h.qos.sim_data;
4647         hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4648             done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4649         queue = &cam_doneqs[hash];
4650         mtx_lock(&queue->cam_doneq_mtx);
4651         run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4652         STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4653         done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4654         mtx_unlock(&queue->cam_doneq_mtx);
4655         if (run)
4656                 wakeup(&queue->cam_doneq);
4657 }
4658
4659 void
4660 xpt_done_direct(union ccb *done_ccb)
4661 {
4662
4663         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
4664             ("xpt_done_direct: status %#x\n", done_ccb->ccb_h.status));
4665         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4666                 return;
4667
4668         /* Store the time the ccb was in the sim */
4669         done_ccb->ccb_h.qos.sim_data = sbinuptime() - done_ccb->ccb_h.qos.sim_data;
4670         xpt_done_process(&done_ccb->ccb_h);
4671 }
4672
4673 union ccb *
4674 xpt_alloc_ccb()
4675 {
4676         union ccb *new_ccb;
4677
4678         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4679         return (new_ccb);
4680 }
4681
4682 union ccb *
4683 xpt_alloc_ccb_nowait()
4684 {
4685         union ccb *new_ccb;
4686
4687         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4688         return (new_ccb);
4689 }
4690
4691 void
4692 xpt_free_ccb(union ccb *free_ccb)
4693 {
4694         free(free_ccb, M_CAMCCB);
4695 }
4696
4697
4698
4699 /* Private XPT functions */
4700
4701 /*
4702  * Get a CAM control block for the caller. Charge the structure to the device
4703  * referenced by the path.  If we don't have sufficient resources to allocate
4704  * more ccbs, we return NULL.
4705  */
4706 static union ccb *
4707 xpt_get_ccb_nowait(struct cam_periph *periph)
4708 {
4709         union ccb *new_ccb;
4710
4711         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4712         if (new_ccb == NULL)
4713                 return (NULL);
4714         periph->periph_allocated++;
4715         cam_ccbq_take_opening(&periph->path->device->ccbq);
4716         return (new_ccb);
4717 }
4718
4719 static union ccb *
4720 xpt_get_ccb(struct cam_periph *periph)
4721 {
4722         union ccb *new_ccb;
4723
4724         cam_periph_unlock(periph);
4725         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4726         cam_periph_lock(periph);
4727         periph->periph_allocated++;
4728         cam_ccbq_take_opening(&periph->path->device->ccbq);
4729         return (new_ccb);
4730 }
4731
4732 union ccb *
4733 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4734 {
4735         struct ccb_hdr *ccb_h;
4736
4737         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4738         cam_periph_assert(periph, MA_OWNED);
4739         while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4740             ccb_h->pinfo.priority != priority) {
4741                 if (priority < periph->immediate_priority) {
4742                         periph->immediate_priority = priority;
4743                         xpt_run_allocq(periph, 0);
4744                 } else
4745                         cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4746                             "cgticb", 0);
4747         }
4748         SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4749         return ((union ccb *)ccb_h);
4750 }
4751
4752 static void
4753 xpt_acquire_bus(struct cam_eb *bus)
4754 {
4755
4756         xpt_lock_buses();
4757         bus->refcount++;
4758         xpt_unlock_buses();
4759 }
4760
4761 static void
4762 xpt_release_bus(struct cam_eb *bus)
4763 {
4764
4765         xpt_lock_buses();
4766         KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4767         if (--bus->refcount > 0) {
4768                 xpt_unlock_buses();
4769                 return;
4770         }
4771         TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4772         xsoftc.bus_generation++;
4773         xpt_unlock_buses();
4774         KASSERT(TAILQ_EMPTY(&bus->et_entries),
4775             ("destroying bus, but target list is not empty"));
4776         cam_sim_release(bus->sim);
4777         mtx_destroy(&bus->eb_mtx);
4778         free(bus, M_CAMXPT);
4779 }
4780
4781 static struct cam_et *
4782 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4783 {
4784         struct cam_et *cur_target, *target;
4785
4786         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4787         mtx_assert(&bus->eb_mtx, MA_OWNED);
4788         target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4789                                          M_NOWAIT|M_ZERO);
4790         if (target == NULL)
4791                 return (NULL);
4792
4793         TAILQ_INIT(&target->ed_entries);
4794         target->bus = bus;
4795         target->target_id = target_id;
4796         target->refcount = 1;
4797         target->generation = 0;
4798         target->luns = NULL;
4799         mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4800         timevalclear(&target->last_reset);
4801         /*
4802          * Hold a reference to our parent bus so it
4803          * will not go away before we do.
4804          */
4805         bus->refcount++;
4806
4807         /* Insertion sort into our bus's target list */
4808         cur_target = TAILQ_FIRST(&bus->et_entries);
4809         while (cur_target != NULL && cur_target->target_id < target_id)
4810                 cur_target = TAILQ_NEXT(cur_target, links);
4811         if (cur_target != NULL) {
4812                 TAILQ_INSERT_BEFORE(cur_target, target, links);
4813         } else {
4814                 TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4815         }
4816         bus->generation++;
4817         return (target);
4818 }
4819
4820 static void
4821 xpt_acquire_target(struct cam_et *target)
4822 {
4823         struct cam_eb *bus = target->bus;
4824
4825         mtx_lock(&bus->eb_mtx);
4826         target->refcount++;
4827         mtx_unlock(&bus->eb_mtx);
4828 }
4829
4830 static void
4831 xpt_release_target(struct cam_et *target)
4832 {
4833         struct cam_eb *bus = target->bus;
4834
4835         mtx_lock(&bus->eb_mtx);
4836         if (--target->refcount > 0) {
4837                 mtx_unlock(&bus->eb_mtx);
4838                 return;
4839         }
4840         TAILQ_REMOVE(&bus->et_entries, target, links);
4841         bus->generation++;
4842         mtx_unlock(&bus->eb_mtx);
4843         KASSERT(TAILQ_EMPTY(&target->ed_entries),
4844             ("destroying target, but device list is not empty"));
4845         xpt_release_bus(bus);
4846         mtx_destroy(&target->luns_mtx);
4847         if (target->luns)
4848                 free(target->luns, M_CAMXPT);
4849         free(target, M_CAMXPT);
4850 }
4851
4852 static struct cam_ed *
4853 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4854                          lun_id_t lun_id)
4855 {
4856         struct cam_ed *device;
4857
4858         device = xpt_alloc_device(bus, target, lun_id);
4859         if (device == NULL)
4860                 return (NULL);
4861
4862         device->mintags = 1;
4863         device->maxtags = 1;
4864         return (device);
4865 }
4866
4867 static void
4868 xpt_destroy_device(void *context, int pending)
4869 {
4870         struct cam_ed   *device = context;
4871
4872         mtx_lock(&device->device_mtx);
4873         mtx_destroy(&device->device_mtx);
4874         free(device, M_CAMDEV);
4875 }
4876
4877 struct cam_ed *
4878 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4879 {
4880         struct cam_ed   *cur_device, *device;
4881         struct cam_devq *devq;
4882         cam_status status;
4883
4884         mtx_assert(&bus->eb_mtx, MA_OWNED);
4885         /* Make space for us in the device queue on our bus */
4886         devq = bus->sim->devq;
4887         mtx_lock(&devq->send_mtx);
4888         status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4889         mtx_unlock(&devq->send_mtx);
4890         if (status != CAM_REQ_CMP)
4891                 return (NULL);
4892
4893         device = (struct cam_ed *)malloc(sizeof(*device),
4894                                          M_CAMDEV, M_NOWAIT|M_ZERO);
4895         if (device == NULL)
4896                 return (NULL);
4897
4898         cam_init_pinfo(&device->devq_entry);
4899         device->target = target;
4900         device->lun_id = lun_id;
4901         device->sim = bus->sim;
4902         if (cam_ccbq_init(&device->ccbq,
4903                           bus->sim->max_dev_openings) != 0) {
4904                 free(device, M_CAMDEV);
4905                 return (NULL);
4906         }
4907         SLIST_INIT(&device->asyncs);
4908         SLIST_INIT(&device->periphs);
4909         device->generation = 0;
4910         device->flags = CAM_DEV_UNCONFIGURED;
4911         device->tag_delay_count = 0;
4912         device->tag_saved_openings = 0;
4913         device->refcount = 1;
4914         mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4915         callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4916         TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4917         /*
4918          * Hold a reference to our parent bus so it
4919          * will not go away before we do.
4920          */
4921         target->refcount++;
4922
4923         cur_device = TAILQ_FIRST(&target->ed_entries);
4924         while (cur_device != NULL && cur_device->lun_id < lun_id)
4925                 cur_device = TAILQ_NEXT(cur_device, links);
4926         if (cur_device != NULL)
4927                 TAILQ_INSERT_BEFORE(cur_device, device, links);
4928         else
4929                 TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4930         target->generation++;
4931         return (device);
4932 }
4933
4934 void
4935 xpt_acquire_device(struct cam_ed *device)
4936 {
4937         struct cam_eb *bus = device->target->bus;
4938
4939         mtx_lock(&bus->eb_mtx);
4940         device->refcount++;
4941         mtx_unlock(&bus->eb_mtx);
4942 }
4943
4944 void
4945 xpt_release_device(struct cam_ed *device)
4946 {
4947         struct cam_eb *bus = device->target->bus;
4948         struct cam_devq *devq;
4949
4950         mtx_lock(&bus->eb_mtx);
4951         if (--device->refcount > 0) {
4952                 mtx_unlock(&bus->eb_mtx);
4953                 return;
4954         }
4955
4956         TAILQ_REMOVE(&device->target->ed_entries, device,links);
4957         device->target->generation++;
4958         mtx_unlock(&bus->eb_mtx);
4959
4960         /* Release our slot in the devq */
4961         devq = bus->sim->devq;
4962         mtx_lock(&devq->send_mtx);
4963         cam_devq_resize(devq, devq->send_queue.array_size - 1);
4964         mtx_unlock(&devq->send_mtx);
4965
4966         KASSERT(SLIST_EMPTY(&device->periphs),
4967             ("destroying device, but periphs list is not empty"));
4968         KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4969             ("destroying device while still queued for ccbs"));
4970
4971         if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4972                 callout_stop(&device->callout);
4973
4974         xpt_release_target(device->target);
4975
4976         cam_ccbq_fini(&device->ccbq);
4977         /*
4978          * Free allocated memory.  free(9) does nothing if the
4979          * supplied pointer is NULL, so it is safe to call without
4980          * checking.
4981          */
4982         free(device->supported_vpds, M_CAMXPT);
4983         free(device->device_id, M_CAMXPT);
4984         free(device->ext_inq, M_CAMXPT);
4985         free(device->physpath, M_CAMXPT);
4986         free(device->rcap_buf, M_CAMXPT);
4987         free(device->serial_num, M_CAMXPT);
4988         taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4989 }
4990
4991 u_int32_t
4992 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4993 {
4994         int     result;
4995         struct  cam_ed *dev;
4996
4997         dev = path->device;
4998         mtx_lock(&dev->sim->devq->send_mtx);
4999         result = cam_ccbq_resize(&dev->ccbq, newopenings);
5000         mtx_unlock(&dev->sim->devq->send_mtx);
5001         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5002          || (dev->inq_flags & SID_CmdQue) != 0)
5003                 dev->tag_saved_openings = newopenings;
5004         return (result);
5005 }
5006
5007 static struct cam_eb *
5008 xpt_find_bus(path_id_t path_id)
5009 {
5010         struct cam_eb *bus;
5011
5012         xpt_lock_buses();
5013         for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
5014              bus != NULL;
5015              bus = TAILQ_NEXT(bus, links)) {
5016                 if (bus->path_id == path_id) {
5017                         bus->refcount++;
5018                         break;
5019                 }
5020         }
5021         xpt_unlock_buses();
5022         return (bus);
5023 }
5024
5025 static struct cam_et *
5026 xpt_find_target(struct cam_eb *bus, target_id_t target_id)
5027 {
5028         struct cam_et *target;
5029
5030         mtx_assert(&bus->eb_mtx, MA_OWNED);
5031         for (target = TAILQ_FIRST(&bus->et_entries);
5032              target != NULL;
5033              target = TAILQ_NEXT(target, links)) {
5034                 if (target->target_id == target_id) {
5035                         target->refcount++;
5036                         break;
5037                 }
5038         }
5039         return (target);
5040 }
5041
5042 static struct cam_ed *
5043 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
5044 {
5045         struct cam_ed *device;
5046
5047         mtx_assert(&target->bus->eb_mtx, MA_OWNED);
5048         for (device = TAILQ_FIRST(&target->ed_entries);
5049              device != NULL;
5050              device = TAILQ_NEXT(device, links)) {
5051                 if (device->lun_id == lun_id) {
5052                         device->refcount++;
5053                         break;
5054                 }
5055         }
5056         return (device);
5057 }
5058
5059 void
5060 xpt_start_tags(struct cam_path *path)
5061 {
5062         struct ccb_relsim crs;
5063         struct cam_ed *device;
5064         struct cam_sim *sim;
5065         int    newopenings;
5066
5067         device = path->device;
5068         sim = path->bus->sim;
5069         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
5070         xpt_freeze_devq(path, /*count*/1);
5071         device->inq_flags |= SID_CmdQue;
5072         if (device->tag_saved_openings != 0)
5073                 newopenings = device->tag_saved_openings;
5074         else
5075                 newopenings = min(device->maxtags,
5076                                   sim->max_tagged_dev_openings);
5077         xpt_dev_ccbq_resize(path, newopenings);
5078         xpt_async(AC_GETDEV_CHANGED, path, NULL);
5079         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
5080         crs.ccb_h.func_code = XPT_REL_SIMQ;
5081         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
5082         crs.openings
5083             = crs.release_timeout
5084             = crs.qfrozen_cnt
5085             = 0;
5086         xpt_action((union ccb *)&crs);
5087 }
5088
5089 void
5090 xpt_stop_tags(struct cam_path *path)
5091 {
5092         struct ccb_relsim crs;
5093         struct cam_ed *device;
5094         struct cam_sim *sim;
5095
5096         device = path->device;
5097         sim = path->bus->sim;
5098         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
5099         device->tag_delay_count = 0;
5100         xpt_freeze_devq(path, /*count*/1);
5101         device->inq_flags &= ~SID_CmdQue;
5102         xpt_dev_ccbq_resize(path, sim->max_dev_openings);
5103         xpt_async(AC_GETDEV_CHANGED, path, NULL);
5104         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
5105         crs.ccb_h.func_code = XPT_REL_SIMQ;
5106         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
5107         crs.openings
5108             = crs.release_timeout
5109             = crs.qfrozen_cnt
5110             = 0;
5111         xpt_action((union ccb *)&crs);
5112 }
5113
5114 static void
5115 xpt_boot_delay(void *arg)
5116 {
5117
5118         xpt_release_boot();
5119 }
5120
5121 static void
5122 xpt_config(void *arg)
5123 {
5124         /*
5125          * Now that interrupts are enabled, go find our devices
5126          */
5127         if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
5128                 printf("xpt_config: failed to create taskqueue thread.\n");
5129
5130         /* Setup debugging path */
5131         if (cam_dflags != CAM_DEBUG_NONE) {
5132                 if (xpt_create_path(&cam_dpath, NULL,
5133                                     CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
5134                                     CAM_DEBUG_LUN) != CAM_REQ_CMP) {
5135                         printf("xpt_config: xpt_create_path() failed for debug"
5136                                " target %d:%d:%d, debugging disabled\n",
5137                                CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
5138                         cam_dflags = CAM_DEBUG_NONE;
5139                 }
5140         } else
5141                 cam_dpath = NULL;
5142
5143         periphdriver_init(1);
5144         xpt_hold_boot();
5145         callout_init(&xsoftc.boot_callout, 1);
5146         callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
5147             xpt_boot_delay, NULL, 0);
5148         /* Fire up rescan thread. */
5149         if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
5150             "cam", "scanner")) {
5151                 printf("xpt_config: failed to create rescan thread.\n");
5152         }
5153 }
5154
5155 void
5156 xpt_hold_boot(void)
5157 {
5158         xpt_lock_buses();
5159         xsoftc.buses_to_config++;
5160         xpt_unlock_buses();
5161 }
5162
5163 void
5164 xpt_release_boot(void)
5165 {
5166         xpt_lock_buses();
5167         xsoftc.buses_to_config--;
5168         if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
5169                 struct  xpt_task *task;
5170
5171                 xsoftc.buses_config_done = 1;
5172                 xpt_unlock_buses();
5173                 /* Call manually because we don't have any buses */
5174                 task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
5175                 if (task != NULL) {
5176                         TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
5177                         taskqueue_enqueue(taskqueue_thread, &task->task);
5178                 }
5179         } else
5180                 xpt_unlock_buses();
5181 }
5182
5183 /*
5184  * If the given device only has one peripheral attached to it, and if that
5185  * peripheral is the passthrough driver, announce it.  This insures that the
5186  * user sees some sort of announcement for every peripheral in their system.
5187  */
5188 static int
5189 xptpassannouncefunc(struct cam_ed *device, void *arg)
5190 {
5191         struct cam_periph *periph;
5192         int i;
5193
5194         for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5195              periph = SLIST_NEXT(periph, periph_links), i++);
5196
5197         periph = SLIST_FIRST(&device->periphs);
5198         if ((i == 1)
5199          && (strncmp(periph->periph_name, "pass", 4) == 0))
5200                 xpt_announce_periph(periph, NULL);
5201
5202         return(1);
5203 }
5204
5205 static void
5206 xpt_finishconfig_task(void *context, int pending)
5207 {
5208
5209         periphdriver_init(2);
5210         /*
5211          * Check for devices with no "standard" peripheral driver
5212          * attached.  For any devices like that, announce the
5213          * passthrough driver so the user will see something.
5214          */
5215         if (!bootverbose)
5216                 xpt_for_all_devices(xptpassannouncefunc, NULL);
5217
5218         /* Release our hook so that the boot can continue. */
5219         config_intrhook_disestablish(xsoftc.xpt_config_hook);
5220         free(xsoftc.xpt_config_hook, M_CAMXPT);
5221         xsoftc.xpt_config_hook = NULL;
5222
5223         free(context, M_CAMXPT);
5224 }
5225
5226 cam_status
5227 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5228                    struct cam_path *path)
5229 {
5230         struct ccb_setasync csa;
5231         cam_status status;
5232         int xptpath = 0;
5233
5234         if (path == NULL) {
5235                 status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5236                                          CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5237                 if (status != CAM_REQ_CMP)
5238                         return (status);
5239                 xpt_path_lock(path);
5240                 xptpath = 1;
5241         }
5242
5243         xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5244         csa.ccb_h.func_code = XPT_SASYNC_CB;
5245         csa.event_enable = event;
5246         csa.callback = cbfunc;
5247         csa.callback_arg = cbarg;
5248         xpt_action((union ccb *)&csa);
5249         status = csa.ccb_h.status;
5250
5251         CAM_DEBUG(csa.ccb_h.path, CAM_DEBUG_TRACE,
5252             ("xpt_register_async: func %p\n", cbfunc));
5253
5254         if (xptpath) {
5255                 xpt_path_unlock(path);
5256                 xpt_free_path(path);
5257         }
5258
5259         if ((status == CAM_REQ_CMP) &&
5260             (csa.event_enable & AC_FOUND_DEVICE)) {
5261                 /*
5262                  * Get this peripheral up to date with all
5263                  * the currently existing devices.
5264                  */
5265                 xpt_for_all_devices(xptsetasyncfunc, &csa);
5266         }
5267         if ((status == CAM_REQ_CMP) &&
5268             (csa.event_enable & AC_PATH_REGISTERED)) {
5269                 /*
5270                  * Get this peripheral up to date with all
5271                  * the currently existing buses.
5272                  */
5273                 xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5274         }
5275
5276         return (status);
5277 }
5278
5279 static void
5280 xptaction(struct cam_sim *sim, union ccb *work_ccb)
5281 {
5282         CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5283
5284         switch (work_ccb->ccb_h.func_code) {
5285         /* Common cases first */
5286         case XPT_PATH_INQ:              /* Path routing inquiry */
5287         {
5288                 struct ccb_pathinq *cpi;
5289
5290                 cpi = &work_ccb->cpi;
5291                 cpi->version_num = 1; /* XXX??? */
5292                 cpi->hba_inquiry = 0;
5293                 cpi->target_sprt = 0;
5294                 cpi->hba_misc = 0;
5295                 cpi->hba_eng_cnt = 0;
5296                 cpi->max_target = 0;
5297                 cpi->max_lun = 0;
5298                 cpi->initiator_id = 0;
5299                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5300                 strlcpy(cpi->hba_vid, "", HBA_IDLEN);
5301                 strlcpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5302                 cpi->unit_number = sim->unit_number;
5303                 cpi->bus_id = sim->bus_id;
5304                 cpi->base_transfer_speed = 0;
5305                 cpi->protocol = PROTO_UNSPECIFIED;
5306                 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5307                 cpi->transport = XPORT_UNSPECIFIED;
5308                 cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5309                 cpi->ccb_h.status = CAM_REQ_CMP;
5310                 xpt_done(work_ccb);
5311                 break;
5312         }
5313         default:
5314                 work_ccb->ccb_h.status = CAM_REQ_INVALID;
5315                 xpt_done(work_ccb);
5316                 break;
5317         }
5318 }
5319
5320 /*
5321  * The xpt as a "controller" has no interrupt sources, so polling
5322  * is a no-op.
5323  */
5324 static void
5325 xptpoll(struct cam_sim *sim)
5326 {
5327 }
5328
5329 void
5330 xpt_lock_buses(void)
5331 {
5332         mtx_lock(&xsoftc.xpt_topo_lock);
5333 }
5334
5335 void
5336 xpt_unlock_buses(void)
5337 {
5338         mtx_unlock(&xsoftc.xpt_topo_lock);
5339 }
5340
5341 struct mtx *
5342 xpt_path_mtx(struct cam_path *path)
5343 {
5344
5345         return (&path->device->device_mtx);
5346 }
5347
5348 static void
5349 xpt_done_process(struct ccb_hdr *ccb_h)
5350 {
5351         struct cam_sim *sim;
5352         struct cam_devq *devq;
5353         struct mtx *mtx = NULL;
5354
5355 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
5356         struct ccb_scsiio *csio;
5357
5358         if (ccb_h->func_code == XPT_SCSI_IO) {
5359                 csio = &((union ccb *)ccb_h)->csio;
5360                 if (csio->bio != NULL)
5361                         biotrack(csio->bio, __func__);
5362         }
5363 #endif
5364
5365         if (ccb_h->flags & CAM_HIGH_POWER) {
5366                 struct highpowerlist    *hphead;
5367                 struct cam_ed           *device;
5368
5369                 mtx_lock(&xsoftc.xpt_highpower_lock);
5370                 hphead = &xsoftc.highpowerq;
5371
5372                 device = STAILQ_FIRST(hphead);
5373
5374                 /*
5375                  * Increment the count since this command is done.
5376                  */
5377                 xsoftc.num_highpower++;
5378
5379                 /*
5380                  * Any high powered commands queued up?
5381                  */
5382                 if (device != NULL) {
5383
5384                         STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5385                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5386
5387                         mtx_lock(&device->sim->devq->send_mtx);
5388                         xpt_release_devq_device(device,
5389                                          /*count*/1, /*runqueue*/TRUE);
5390                         mtx_unlock(&device->sim->devq->send_mtx);
5391                 } else
5392                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5393         }
5394
5395         sim = ccb_h->path->bus->sim;
5396
5397         if (ccb_h->status & CAM_RELEASE_SIMQ) {
5398                 xpt_release_simq(sim, /*run_queue*/FALSE);
5399                 ccb_h->status &= ~CAM_RELEASE_SIMQ;
5400         }
5401
5402         if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5403          && (ccb_h->status & CAM_DEV_QFRZN)) {
5404                 xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5405                 ccb_h->status &= ~CAM_DEV_QFRZN;
5406         }
5407
5408         devq = sim->devq;
5409         if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5410                 struct cam_ed *dev = ccb_h->path->device;
5411
5412                 mtx_lock(&devq->send_mtx);
5413                 devq->send_active--;
5414                 devq->send_openings++;
5415                 cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5416
5417                 if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5418                   && (dev->ccbq.dev_active == 0))) {
5419                         dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5420                         xpt_release_devq_device(dev, /*count*/1,
5421                                          /*run_queue*/FALSE);
5422                 }
5423
5424                 if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5425                   && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5426                         dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5427                         xpt_release_devq_device(dev, /*count*/1,
5428                                          /*run_queue*/FALSE);
5429                 }
5430
5431                 if (!device_is_queued(dev))
5432                         (void)xpt_schedule_devq(devq, dev);
5433                 xpt_run_devq(devq);
5434                 mtx_unlock(&devq->send_mtx);
5435
5436                 if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5437                         mtx = xpt_path_mtx(ccb_h->path);
5438                         mtx_lock(mtx);
5439
5440                         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5441                          && (--dev->tag_delay_count == 0))
5442                                 xpt_start_tags(ccb_h->path);
5443                 }
5444         }
5445
5446         if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5447                 if (mtx == NULL) {
5448                         mtx = xpt_path_mtx(ccb_h->path);
5449                         mtx_lock(mtx);
5450                 }
5451         } else {
5452                 if (mtx != NULL) {
5453                         mtx_unlock(mtx);
5454                         mtx = NULL;
5455                 }
5456         }
5457
5458         /* Call the peripheral driver's callback */
5459         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5460         (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5461         if (mtx != NULL)
5462                 mtx_unlock(mtx);
5463 }
5464
5465 void
5466 xpt_done_td(void *arg)
5467 {
5468         struct cam_doneq *queue = arg;
5469         struct ccb_hdr *ccb_h;
5470         STAILQ_HEAD(, ccb_hdr)  doneq;
5471
5472         STAILQ_INIT(&doneq);
5473         mtx_lock(&queue->cam_doneq_mtx);
5474         while (1) {
5475                 while (STAILQ_EMPTY(&queue->cam_doneq)) {
5476                         queue->cam_doneq_sleep = 1;
5477                         msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5478                             PRIBIO, "-", 0);
5479                         queue->cam_doneq_sleep = 0;
5480                 }
5481                 STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5482                 mtx_unlock(&queue->cam_doneq_mtx);
5483
5484                 THREAD_NO_SLEEPING();
5485                 while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5486                         STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5487                         xpt_done_process(ccb_h);
5488                 }
5489                 THREAD_SLEEPING_OK();
5490
5491                 mtx_lock(&queue->cam_doneq_mtx);
5492         }
5493 }
5494
5495 static void
5496 camisr_runqueue(void)
5497 {
5498         struct  ccb_hdr *ccb_h;
5499         struct cam_doneq *queue;
5500         int i;
5501
5502         /* Process global queues. */
5503         for (i = 0; i < cam_num_doneqs; i++) {
5504                 queue = &cam_doneqs[i];
5505                 mtx_lock(&queue->cam_doneq_mtx);
5506                 while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5507                         STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5508                         mtx_unlock(&queue->cam_doneq_mtx);
5509                         xpt_done_process(ccb_h);
5510                         mtx_lock(&queue->cam_doneq_mtx);
5511                 }
5512                 mtx_unlock(&queue->cam_doneq_mtx);
5513         }
5514 }
5515
5516 struct kv
5517 {
5518         uint32_t v;
5519         const char *name;
5520 };
5521
5522 static struct kv map[] = {
5523         { XPT_NOOP, "XPT_NOOP" },
5524         { XPT_SCSI_IO, "XPT_SCSI_IO" },
5525         { XPT_GDEV_TYPE, "XPT_GDEV_TYPE" },
5526         { XPT_GDEVLIST, "XPT_GDEVLIST" },
5527         { XPT_PATH_INQ, "XPT_PATH_INQ" },
5528         { XPT_REL_SIMQ, "XPT_REL_SIMQ" },
5529         { XPT_SASYNC_CB, "XPT_SASYNC_CB" },
5530         { XPT_SDEV_TYPE, "XPT_SDEV_TYPE" },
5531         { XPT_SCAN_BUS, "XPT_SCAN_BUS" },
5532         { XPT_DEV_MATCH, "XPT_DEV_MATCH" },
5533         { XPT_DEBUG, "XPT_DEBUG" },
5534         { XPT_PATH_STATS, "XPT_PATH_STATS" },
5535         { XPT_GDEV_STATS, "XPT_GDEV_STATS" },
5536         { XPT_DEV_ADVINFO, "XPT_DEV_ADVINFO" },
5537         { XPT_ASYNC, "XPT_ASYNC" },
5538         { XPT_ABORT, "XPT_ABORT" },
5539         { XPT_RESET_BUS, "XPT_RESET_BUS" },
5540         { XPT_RESET_DEV, "XPT_RESET_DEV" },
5541         { XPT_TERM_IO, "XPT_TERM_IO" },
5542         { XPT_SCAN_LUN, "XPT_SCAN_LUN" },
5543         { XPT_GET_TRAN_SETTINGS, "XPT_GET_TRAN_SETTINGS" },
5544         { XPT_SET_TRAN_SETTINGS, "XPT_SET_TRAN_SETTINGS" },
5545         { XPT_CALC_GEOMETRY, "XPT_CALC_GEOMETRY" },
5546         { XPT_ATA_IO, "XPT_ATA_IO" },
5547         { XPT_GET_SIM_KNOB, "XPT_GET_SIM_KNOB" },
5548         { XPT_SET_SIM_KNOB, "XPT_SET_SIM_KNOB" },
5549         { XPT_NVME_IO, "XPT_NVME_IO" },
5550         { XPT_MMC_IO, "XPT_MMC_IO" },
5551         { XPT_SMP_IO, "XPT_SMP_IO" },
5552         { XPT_SCAN_TGT, "XPT_SCAN_TGT" },
5553         { XPT_ENG_INQ, "XPT_ENG_INQ" },
5554         { XPT_ENG_EXEC, "XPT_ENG_EXEC" },
5555         { XPT_EN_LUN, "XPT_EN_LUN" },
5556         { XPT_TARGET_IO, "XPT_TARGET_IO" },
5557         { XPT_ACCEPT_TARGET_IO, "XPT_ACCEPT_TARGET_IO" },
5558         { XPT_CONT_TARGET_IO, "XPT_CONT_TARGET_IO" },
5559         { XPT_IMMED_NOTIFY, "XPT_IMMED_NOTIFY" },
5560         { XPT_NOTIFY_ACK, "XPT_NOTIFY_ACK" },
5561         { XPT_IMMEDIATE_NOTIFY, "XPT_IMMEDIATE_NOTIFY" },
5562         { XPT_NOTIFY_ACKNOWLEDGE, "XPT_NOTIFY_ACKNOWLEDGE" },
5563         { 0, 0 }
5564 };
5565
5566 const char *
5567 xpt_action_name(uint32_t action)
5568 {
5569         static char buffer[32]; /* Only for unknown messages -- racy */
5570         struct kv *walker = map;
5571
5572         while (walker->name != NULL) {
5573                 if (walker->v == action)
5574                         return (walker->name);
5575                 walker++;
5576         }
5577
5578         snprintf(buffer, sizeof(buffer), "%#x", action);
5579         return (buffer);
5580 }