sys/cam/cam_xpt.c

   1 /*-
   2  * Implementation of the Common Access Method Transport (XPT) layer.
   3  *
   4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
   5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification, immediately at the beginning of the file.
  14  * 2. The name of the author may not be used to endorse or promote products
  15  *    derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/bus.h>
  35 #include <sys/systm.h>
  36 #include <sys/types.h>
  37 #include <sys/malloc.h>
  38 #include <sys/kernel.h>
  39 #include <sys/time.h>
  40 #include <sys/conf.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/interrupt.h>
  43 #include <sys/proc.h>
  44 #include <sys/sbuf.h>
  45 #include <sys/smp.h>
  46 #include <sys/taskqueue.h>
  47
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/sysctl.h>
  51 #include <sys/kthread.h>
  52
  53 #include <cam/cam.h>
  54 #include <cam/cam_ccb.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/cam_queue.h>
  57 #include <cam/cam_sim.h>
  58 #include <cam/cam_xpt.h>
  59 #include <cam/cam_xpt_sim.h>
  60 #include <cam/cam_xpt_periph.h>
  61 #include <cam/cam_xpt_internal.h>
  62 #include <cam/cam_debug.h>
  63 #include <cam/cam_compat.h>
  64
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_message.h>
  67 #include <cam/scsi/scsi_pass.h>
  68
  69 #include <machine/md_var.h>     /* geometry translation */
  70 #include <machine/stdarg.h>     /* for xpt_print below */
  71
  72 #include "opt_cam.h"
  73
  74 /*
  75  * This is the maximum number of high powered commands (e.g. start unit)
  76  * that can be outstanding at a particular time.
  77  */
  78 #ifndef CAM_MAX_HIGHPOWER
  79 #define CAM_MAX_HIGHPOWER  4
  80 #endif
  81
  82 /* Datastructures internal to the xpt layer */
  83 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
  84 MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
  85 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
  86 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
  87
  88 /* Object for defering XPT actions to a taskqueue */
  89 struct xpt_task {
  90         struct task     task;
  91         void            *data1;
  92         uintptr_t       data2;
  93 };
  94
  95 struct xpt_softc {
  96         uint32_t                xpt_generation;
  97
  98         /* number of high powered commands that can go through right now */
  99         struct mtx              xpt_highpower_lock;
 100         STAILQ_HEAD(highpowerlist, cam_ed)      highpowerq;
 101         int                     num_highpower;
 102
 103         /* queue for handling async rescan requests. */
 104         TAILQ_HEAD(, ccb_hdr) ccb_scanq;
 105         int buses_to_config;
 106         int buses_config_done;
 107
 108         /* Registered busses */
 109         TAILQ_HEAD(,cam_eb)     xpt_busses;
 110         u_int                   bus_generation;
 111
 112         struct intr_config_hook *xpt_config_hook;
 113
 114         int                     boot_delay;
 115         struct callout          boot_callout;
 116
 117         struct mtx              xpt_topo_lock;
 118         struct mtx              xpt_lock;
 119         struct taskqueue        *xpt_taskq;
 120 };
 121
 122 typedef enum {
 123         DM_RET_COPY             = 0x01,
 124         DM_RET_FLAG_MASK        = 0x0f,
 125         DM_RET_NONE             = 0x00,
 126         DM_RET_STOP             = 0x10,
 127         DM_RET_DESCEND          = 0x20,
 128         DM_RET_ERROR            = 0x30,
 129         DM_RET_ACTION_MASK      = 0xf0
 130 } dev_match_ret;
 131
 132 typedef enum {
 133         XPT_DEPTH_BUS,
 134         XPT_DEPTH_TARGET,
 135         XPT_DEPTH_DEVICE,
 136         XPT_DEPTH_PERIPH
 137 } xpt_traverse_depth;
 138
 139 struct xpt_traverse_config {
 140         xpt_traverse_depth      depth;
 141         void                    *tr_func;
 142         void                    *tr_arg;
 143 };
 144
 145 typedef int     xpt_busfunc_t (struct cam_eb *bus, void *arg);
 146 typedef int     xpt_targetfunc_t (struct cam_et *target, void *arg);
 147 typedef int     xpt_devicefunc_t (struct cam_ed *device, void *arg);
 148 typedef int     xpt_periphfunc_t (struct cam_periph *periph, void *arg);
 149 typedef int     xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
 150
 151 /* Transport layer configuration information */
 152 static struct xpt_softc xsoftc;
 153
 154 MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
 155
 156 TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
 157 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
 158            &xsoftc.boot_delay, 0, "Bus registration wait time");
 159 SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
 160             &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
 161
 162 struct cam_doneq {
 163         struct mtx_padalign     cam_doneq_mtx;
 164         STAILQ_HEAD(, ccb_hdr)  cam_doneq;
 165         int                     cam_doneq_sleep;
 166 };
 167
 168 static struct cam_doneq cam_doneqs[MAXCPU];
 169 static int cam_num_doneqs;
 170 static struct proc *cam_proc;
 171
 172 TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs);
 173 SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
 174            &cam_num_doneqs, 0, "Number of completion queues/threads");
 175
 176 struct cam_periph *xpt_periph;
 177
 178 static periph_init_t xpt_periph_init;
 179
 180 static struct periph_driver xpt_driver =
 181 {
 182         xpt_periph_init, "xpt",
 183         TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
 184         CAM_PERIPH_DRV_EARLY
 185 };
 186
 187 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
 188
 189 static d_open_t xptopen;
 190 static d_close_t xptclose;
 191 static d_ioctl_t xptioctl;
 192 static d_ioctl_t xptdoioctl;
 193
 194 static struct cdevsw xpt_cdevsw = {
 195         .d_version =    D_VERSION,
 196         .d_flags =      0,
 197         .d_open =       xptopen,
 198         .d_close =      xptclose,
 199         .d_ioctl =      xptioctl,
 200         .d_name =       "xpt",
 201 };
 202
 203 /* Storage for debugging datastructures */
 204 struct cam_path *cam_dpath;
 205 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
 206 TUNABLE_INT("kern.cam.dflags", &cam_dflags);
 207 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
 208         &cam_dflags, 0, "Enabled debug flags");
 209 u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
 210 TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
 211 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
 212         &cam_debug_delay, 0, "Delay in us after each debug message");
 213
 214 /* Our boot-time initialization hook */
 215 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 216
 217 static moduledata_t cam_moduledata = {
 218         "cam",
 219         cam_module_event_handler,
 220         NULL
 221 };
 222
 223 static int      xpt_init(void *);
 224
 225 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
 226 MODULE_VERSION(cam, 1);
 227
 228
 229 static void             xpt_async_bcast(struct async_list *async_head,
 230                                         u_int32_t async_code,
 231                                         struct cam_path *path,
 232                                         void *async_arg);
 233 static path_id_t xptnextfreepathid(void);
 234 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
 235 static union ccb *xpt_get_ccb(struct cam_periph *periph);
 236 static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
 237 static void      xpt_run_allocq(struct cam_periph *periph, int sleep);
 238 static void      xpt_run_allocq_task(void *context, int pending);
 239 static void      xpt_run_devq(struct cam_devq *devq);
 240 static timeout_t xpt_release_devq_timeout;
 241 static void      xpt_release_simq_timeout(void *arg) __unused;
 242 static void      xpt_acquire_bus(struct cam_eb *bus);
 243 static void      xpt_release_bus(struct cam_eb *bus);
 244 static uint32_t  xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
 245 static int       xpt_release_devq_device(struct cam_ed *dev, u_int count,
 246                     int run_queue);
 247 static struct cam_et*
 248                  xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
 249 static void      xpt_acquire_target(struct cam_et *target);
 250 static void      xpt_release_target(struct cam_et *target);
 251 static struct cam_eb*
 252                  xpt_find_bus(path_id_t path_id);
 253 static struct cam_et*
 254                  xpt_find_target(struct cam_eb *bus, target_id_t target_id);
 255 static struct cam_ed*
 256                  xpt_find_device(struct cam_et *target, lun_id_t lun_id);
 257 static void      xpt_config(void *arg);
 258 static int       xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 259                                  u_int32_t new_priority);
 260 static xpt_devicefunc_t xptpassannouncefunc;
 261 static void      xptaction(struct cam_sim *sim, union ccb *work_ccb);
 262 static void      xptpoll(struct cam_sim *sim);
 263 static void      camisr_runqueue(void);
 264 static void      xpt_done_process(struct ccb_hdr *ccb_h);
 265 static void      xpt_done_td(void *);
 266 static dev_match_ret    xptbusmatch(struct dev_match_pattern *patterns,
 267                                     u_int num_patterns, struct cam_eb *bus);
 268 static dev_match_ret    xptdevicematch(struct dev_match_pattern *patterns,
 269                                        u_int num_patterns,
 270                                        struct cam_ed *device);
 271 static dev_match_ret    xptperiphmatch(struct dev_match_pattern *patterns,
 272                                        u_int num_patterns,
 273                                        struct cam_periph *periph);
 274 static xpt_busfunc_t    xptedtbusfunc;
 275 static xpt_targetfunc_t xptedttargetfunc;
 276 static xpt_devicefunc_t xptedtdevicefunc;
 277 static xpt_periphfunc_t xptedtperiphfunc;
 278 static xpt_pdrvfunc_t   xptplistpdrvfunc;
 279 static xpt_periphfunc_t xptplistperiphfunc;
 280 static int              xptedtmatch(struct ccb_dev_match *cdm);
 281 static int              xptperiphlistmatch(struct ccb_dev_match *cdm);
 282 static int              xptbustraverse(struct cam_eb *start_bus,
 283                                        xpt_busfunc_t *tr_func, void *arg);
 284 static int              xpttargettraverse(struct cam_eb *bus,
 285                                           struct cam_et *start_target,
 286                                           xpt_targetfunc_t *tr_func, void *arg);
 287 static int              xptdevicetraverse(struct cam_et *target,
 288                                           struct cam_ed *start_device,
 289                                           xpt_devicefunc_t *tr_func, void *arg);
 290 static int              xptperiphtraverse(struct cam_ed *device,
 291                                           struct cam_periph *start_periph,
 292                                           xpt_periphfunc_t *tr_func, void *arg);
 293 static int              xptpdrvtraverse(struct periph_driver **start_pdrv,
 294                                         xpt_pdrvfunc_t *tr_func, void *arg);
 295 static int              xptpdperiphtraverse(struct periph_driver **pdrv,
 296                                             struct cam_periph *start_periph,
 297                                             xpt_periphfunc_t *tr_func,
 298                                             void *arg);
 299 static xpt_busfunc_t    xptdefbusfunc;
 300 static xpt_targetfunc_t xptdeftargetfunc;
 301 static xpt_devicefunc_t xptdefdevicefunc;
 302 static xpt_periphfunc_t xptdefperiphfunc;
 303 static void             xpt_finishconfig_task(void *context, int pending);
 304 static void             xpt_dev_async_default(u_int32_t async_code,
 305                                               struct cam_eb *bus,
 306                                               struct cam_et *target,
 307                                               struct cam_ed *device,
 308                                               void *async_arg);
 309 static struct cam_ed *  xpt_alloc_device_default(struct cam_eb *bus,
 310                                                  struct cam_et *target,
 311                                                  lun_id_t lun_id);
 312 static xpt_devicefunc_t xptsetasyncfunc;
 313 static xpt_busfunc_t    xptsetasyncbusfunc;
 314 static cam_status       xptregister(struct cam_periph *periph,
 315                                     void *arg);
 316 static __inline int device_is_queued(struct cam_ed *device);
 317
 318 static __inline int
 319 xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
 320 {
 321         int     retval;
 322
 323         mtx_assert(&devq->send_mtx, MA_OWNED);
 324         if ((dev->ccbq.queue.entries > 0) &&
 325             (dev->ccbq.dev_openings > 0) &&
 326             (dev->ccbq.queue.qfrozen_cnt == 0)) {
 327                 /*
 328                  * The priority of a device waiting for controller
 329                  * resources is that of the highest priority CCB
 330                  * enqueued.
 331                  */
 332                 retval =
 333                     xpt_schedule_dev(&devq->send_queue,
 334                                      &dev->devq_entry,
 335                                      CAMQ_GET_PRIO(&dev->ccbq.queue));
 336         } else {
 337                 retval = 0;
 338         }
 339         return (retval);
 340 }
 341
 342 static __inline int
 343 device_is_queued(struct cam_ed *device)
 344 {
 345         return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
 346 }
 347
 348 static void
 349 xpt_periph_init()
 350 {
 351         make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
 352 }
 353
 354 static int
 355 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 356 {
 357
 358         /*
 359          * Only allow read-write access.
 360          */
 361         if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
 362                 return(EPERM);
 363
 364         /*
 365          * We don't allow nonblocking access.
 366          */
 367         if ((flags & O_NONBLOCK) != 0) {
 368                 printf("%s: can't do nonblocking access\n", devtoname(dev));
 369                 return(ENODEV);
 370         }
 371
 372         return(0);
 373 }
 374
 375 static int
 376 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 377 {
 378
 379         return(0);
 380 }
 381
 382 /*
 383  * Don't automatically grab the xpt softc lock here even though this is going
 384  * through the xpt device.  The xpt device is really just a back door for
 385  * accessing other devices and SIMs, so the right thing to do is to grab
 386  * the appropriate SIM lock once the bus/SIM is located.
 387  */
 388 static int
 389 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 390 {
 391         int error;
 392
 393         if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
 394                 error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
 395         }
 396         return (error);
 397 }
 398
 399 static int
 400 xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 401 {
 402         int error;
 403
 404         error = 0;
 405
 406         switch(cmd) {
 407         /*
 408          * For the transport layer CAMIOCOMMAND ioctl, we really only want
 409          * to accept CCB types that don't quite make sense to send through a
 410          * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
 411          * in the CAM spec.
 412          */
 413         case CAMIOCOMMAND: {
 414                 union ccb *ccb;
 415                 union ccb *inccb;
 416                 struct cam_eb *bus;
 417
 418                 inccb = (union ccb *)addr;
 419
 420                 bus = xpt_find_bus(inccb->ccb_h.path_id);
 421                 if (bus == NULL)
 422                         return (EINVAL);
 423
 424                 switch (inccb->ccb_h.func_code) {
 425                 case XPT_SCAN_BUS:
 426                 case XPT_RESET_BUS:
 427                         if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
 428                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 429                                 xpt_release_bus(bus);
 430                                 return (EINVAL);
 431                         }
 432                         break;
 433                 case XPT_SCAN_TGT:
 434                         if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
 435                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 436                                 xpt_release_bus(bus);
 437                                 return (EINVAL);
 438                         }
 439                         break;
 440                 default:
 441                         break;
 442                 }
 443
 444                 switch(inccb->ccb_h.func_code) {
 445                 case XPT_SCAN_BUS:
 446                 case XPT_RESET_BUS:
 447                 case XPT_PATH_INQ:
 448                 case XPT_ENG_INQ:
 449                 case XPT_SCAN_LUN:
 450                 case XPT_SCAN_TGT:
 451
 452                         ccb = xpt_alloc_ccb();
 453
 454                         /*
 455                          * Create a path using the bus, target, and lun the
 456                          * user passed in.
 457                          */
 458                         if (xpt_create_path(&ccb->ccb_h.path, NULL,
 459                                             inccb->ccb_h.path_id,
 460                                             inccb->ccb_h.target_id,
 461                                             inccb->ccb_h.target_lun) !=
 462                                             CAM_REQ_CMP){
 463                                 error = EINVAL;
 464                                 xpt_free_ccb(ccb);
 465                                 break;
 466                         }
 467                         /* Ensure all of our fields are correct */
 468                         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
 469                                       inccb->ccb_h.pinfo.priority);
 470                         xpt_merge_ccb(ccb, inccb);
 471                         xpt_path_lock(ccb->ccb_h.path);
 472                         cam_periph_runccb(ccb, NULL, 0, 0, NULL);
 473                         xpt_path_unlock(ccb->ccb_h.path);
 474                         bcopy(ccb, inccb, sizeof(union ccb));
 475                         xpt_free_path(ccb->ccb_h.path);
 476                         xpt_free_ccb(ccb);
 477                         break;
 478
 479                 case XPT_DEBUG: {
 480                         union ccb ccb;
 481
 482                         /*
 483                          * This is an immediate CCB, so it's okay to
 484                          * allocate it on the stack.
 485                          */
 486
 487                         /*
 488                          * Create a path using the bus, target, and lun the
 489                          * user passed in.
 490                          */
 491                         if (xpt_create_path(&ccb.ccb_h.path, NULL,
 492                                             inccb->ccb_h.path_id,
 493                                             inccb->ccb_h.target_id,
 494                                             inccb->ccb_h.target_lun) !=
 495                                             CAM_REQ_CMP){
 496                                 error = EINVAL;
 497                                 break;
 498                         }
 499                         /* Ensure all of our fields are correct */
 500                         xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
 501                                       inccb->ccb_h.pinfo.priority);
 502                         xpt_merge_ccb(&ccb, inccb);
 503                         xpt_action(&ccb);
 504                         bcopy(&ccb, inccb, sizeof(union ccb));
 505                         xpt_free_path(ccb.ccb_h.path);
 506                         break;
 507
 508                 }
 509                 case XPT_DEV_MATCH: {
 510                         struct cam_periph_map_info mapinfo;
 511                         struct cam_path *old_path;
 512
 513                         /*
 514                          * We can't deal with physical addresses for this
 515                          * type of transaction.
 516                          */
 517                         if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
 518                             CAM_DATA_VADDR) {
 519                                 error = EINVAL;
 520                                 break;
 521                         }
 522
 523                         /*
 524                          * Save this in case the caller had it set to
 525                          * something in particular.
 526                          */
 527                         old_path = inccb->ccb_h.path;
 528
 529                         /*
 530                          * We really don't need a path for the matching
 531                          * code.  The path is needed because of the
 532                          * debugging statements in xpt_action().  They
 533                          * assume that the CCB has a valid path.
 534                          */
 535                         inccb->ccb_h.path = xpt_periph->path;
 536
 537                         bzero(&mapinfo, sizeof(mapinfo));
 538
 539                         /*
 540                          * Map the pattern and match buffers into kernel
 541                          * virtual address space.
 542                          */
 543                         error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
 544
 545                         if (error) {
 546                                 inccb->ccb_h.path = old_path;
 547                                 break;
 548                         }
 549
 550                         /*
 551                          * This is an immediate CCB, we can send it on directly.
 552                          */
 553                         xpt_action(inccb);
 554
 555                         /*
 556                          * Map the buffers back into user space.
 557                          */
 558                         cam_periph_unmapmem(inccb, &mapinfo);
 559
 560                         inccb->ccb_h.path = old_path;
 561
 562                         error = 0;
 563                         break;
 564                 }
 565                 default:
 566                         error = ENOTSUP;
 567                         break;
 568                 }
 569                 xpt_release_bus(bus);
 570                 break;
 571         }
 572         /*
 573          * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
 574          * with the periphal driver name and unit name filled in.  The other
 575          * fields don't really matter as input.  The passthrough driver name
 576          * ("pass"), and unit number are passed back in the ccb.  The current
 577          * device generation number, and the index into the device peripheral
 578          * driver list, and the status are also passed back.  Note that
 579          * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
 580          * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
 581          * (or rather should be) impossible for the device peripheral driver
 582          * list to change since we look at the whole thing in one pass, and
 583          * we do it with lock protection.
 584          *
 585          */
 586         case CAMGETPASSTHRU: {
 587                 union ccb *ccb;
 588                 struct cam_periph *periph;
 589                 struct periph_driver **p_drv;
 590                 char   *name;
 591                 u_int unit;
 592                 int base_periph_found;
 593
 594                 ccb = (union ccb *)addr;
 595                 unit = ccb->cgdl.unit_number;
 596                 name = ccb->cgdl.periph_name;
 597                 base_periph_found = 0;
 598
 599                 /*
 600                  * Sanity check -- make sure we don't get a null peripheral
 601                  * driver name.
 602                  */
 603                 if (*ccb->cgdl.periph_name == '\0') {
 604                         error = EINVAL;
 605                         break;
 606                 }
 607
 608                 /* Keep the list from changing while we traverse it */
 609                 xpt_lock_buses();
 610
 611                 /* first find our driver in the list of drivers */
 612                 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
 613                         if (strcmp((*p_drv)->driver_name, name) == 0)
 614                                 break;
 615
 616                 if (*p_drv == NULL) {
 617                         xpt_unlock_buses();
 618                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 619                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 620                         *ccb->cgdl.periph_name = '\0';
 621                         ccb->cgdl.unit_number = 0;
 622                         error = ENOENT;
 623                         break;
 624                 }
 625
 626                 /*
 627                  * Run through every peripheral instance of this driver
 628                  * and check to see whether it matches the unit passed
 629                  * in by the user.  If it does, get out of the loops and
 630                  * find the passthrough driver associated with that
 631                  * peripheral driver.
 632                  */
 633                 for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
 634                      periph = TAILQ_NEXT(periph, unit_links)) {
 635
 636                         if (periph->unit_number == unit)
 637                                 break;
 638                 }
 639                 /*
 640                  * If we found the peripheral driver that the user passed
 641                  * in, go through all of the peripheral drivers for that
 642                  * particular device and look for a passthrough driver.
 643                  */
 644                 if (periph != NULL) {
 645                         struct cam_ed *device;
 646                         int i;
 647
 648                         base_periph_found = 1;
 649                         device = periph->path->device;
 650                         for (i = 0, periph = SLIST_FIRST(&device->periphs);
 651                              periph != NULL;
 652                              periph = SLIST_NEXT(periph, periph_links), i++) {
 653                                 /*
 654                                  * Check to see whether we have a
 655                                  * passthrough device or not.
 656                                  */
 657                                 if (strcmp(periph->periph_name, "pass") == 0) {
 658                                         /*
 659                                          * Fill in the getdevlist fields.
 660                                          */
 661                                         strcpy(ccb->cgdl.periph_name,
 662                                                periph->periph_name);
 663                                         ccb->cgdl.unit_number =
 664                                                 periph->unit_number;
 665                                         if (SLIST_NEXT(periph, periph_links))
 666                                                 ccb->cgdl.status =
 667                                                         CAM_GDEVLIST_MORE_DEVS;
 668                                         else
 669                                                 ccb->cgdl.status =
 670                                                        CAM_GDEVLIST_LAST_DEVICE;
 671                                         ccb->cgdl.generation =
 672                                                 device->generation;
 673                                         ccb->cgdl.index = i;
 674                                         /*
 675                                          * Fill in some CCB header fields
 676                                          * that the user may want.
 677                                          */
 678                                         ccb->ccb_h.path_id =
 679                                                 periph->path->bus->path_id;
 680                                         ccb->ccb_h.target_id =
 681                                                 periph->path->target->target_id;
 682                                         ccb->ccb_h.target_lun =
 683                                                 periph->path->device->lun_id;
 684                                         ccb->ccb_h.status = CAM_REQ_CMP;
 685                                         break;
 686                                 }
 687                         }
 688                 }
 689
 690                 /*
 691                  * If the periph is null here, one of two things has
 692                  * happened.  The first possibility is that we couldn't
 693                  * find the unit number of the particular peripheral driver
 694                  * that the user is asking about.  e.g. the user asks for
 695                  * the passthrough driver for "da11".  We find the list of
 696                  * "da" peripherals all right, but there is no unit 11.
 697                  * The other possibility is that we went through the list
 698                  * of peripheral drivers attached to the device structure,
 699                  * but didn't find one with the name "pass".  Either way,
 700                  * we return ENOENT, since we couldn't find something.
 701                  */
 702                 if (periph == NULL) {
 703                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 704                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 705                         *ccb->cgdl.periph_name = '\0';
 706                         ccb->cgdl.unit_number = 0;
 707                         error = ENOENT;
 708                         /*
 709                          * It is unfortunate that this is even necessary,
 710                          * but there are many, many clueless users out there.
 711                          * If this is true, the user is looking for the
 712                          * passthrough driver, but doesn't have one in his
 713                          * kernel.
 714                          */
 715                         if (base_periph_found == 1) {
 716                                 printf("xptioctl: pass driver is not in the "
 717                                        "kernel\n");
 718                                 printf("xptioctl: put \"device pass\" in "
 719                                        "your kernel config file\n");
 720                         }
 721                 }
 722                 xpt_unlock_buses();
 723                 break;
 724                 }
 725         default:
 726                 error = ENOTTY;
 727                 break;
 728         }
 729
 730         return(error);
 731 }
 732
 733 static int
 734 cam_module_event_handler(module_t mod, int what, void *arg)
 735 {
 736         int error;
 737
 738         switch (what) {
 739         case MOD_LOAD:
 740                 if ((error = xpt_init(NULL)) != 0)
 741                         return (error);
 742                 break;
 743         case MOD_UNLOAD:
 744                 return EBUSY;
 745         default:
 746                 return EOPNOTSUPP;
 747         }
 748
 749         return 0;
 750 }
 751
 752 static void
 753 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
 754 {
 755
 756         if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
 757                 xpt_free_path(done_ccb->ccb_h.path);
 758                 xpt_free_ccb(done_ccb);
 759         } else {
 760                 done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
 761                 (*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
 762         }
 763         xpt_release_boot();
 764 }
 765
 766 /* thread to handle bus rescans */
 767 static void
 768 xpt_scanner_thread(void *dummy)
 769 {
 770         union ccb       *ccb;
 771         struct cam_path  path;
 772
 773         xpt_lock_buses();
 774         for (;;) {
 775                 if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
 776                         msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
 777                                "-", 0);
 778                 if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
 779                         TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 780                         xpt_unlock_buses();
 781
 782                         /*
 783                          * Since lock can be dropped inside and path freed
 784                          * by completion callback even before return here,
 785                          * take our own path copy for reference.
 786                          */
 787                         xpt_copy_path(&path, ccb->ccb_h.path);
 788                         xpt_path_lock(&path);
 789                         xpt_action(ccb);
 790                         xpt_path_unlock(&path);
 791                         xpt_release_path(&path);
 792
 793                         xpt_lock_buses();
 794                 }
 795         }
 796 }
 797
 798 void
 799 xpt_rescan(union ccb *ccb)
 800 {
 801         struct ccb_hdr *hdr;
 802
 803         /* Prepare request */
 804         if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
 805             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 806                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 807         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 808             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 809                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 810         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 811             ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
 812                 ccb->ccb_h.func_code = XPT_SCAN_LUN;
 813         else {
 814                 xpt_print(ccb->ccb_h.path, "illegal scan path\n");
 815                 xpt_free_path(ccb->ccb_h.path);
 816                 xpt_free_ccb(ccb);
 817                 return;
 818         }
 819         ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
 820         ccb->ccb_h.cbfcnp = xpt_rescan_done;
 821         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
 822         /* Don't make duplicate entries for the same paths. */
 823         xpt_lock_buses();
 824         if (ccb->ccb_h.ppriv_ptr1 == NULL) {
 825                 TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
 826                         if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
 827                                 wakeup(&xsoftc.ccb_scanq);
 828                                 xpt_unlock_buses();
 829                                 xpt_print(ccb->ccb_h.path, "rescan already queued\n");
 830                                 xpt_free_path(ccb->ccb_h.path);
 831                                 xpt_free_ccb(ccb);
 832                                 return;
 833                         }
 834                 }
 835         }
 836         TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 837         xsoftc.buses_to_config++;
 838         wakeup(&xsoftc.ccb_scanq);
 839         xpt_unlock_buses();
 840 }
 841
 842 /* Functions accessed by the peripheral drivers */
 843 static int
 844 xpt_init(void *dummy)
 845 {
 846         struct cam_sim *xpt_sim;
 847         struct cam_path *path;
 848         struct cam_devq *devq;
 849         cam_status status;
 850         int error, i;
 851
 852         TAILQ_INIT(&xsoftc.xpt_busses);
 853         TAILQ_INIT(&xsoftc.ccb_scanq);
 854         STAILQ_INIT(&xsoftc.highpowerq);
 855         xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
 856
 857         mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
 858         mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
 859         xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
 860             taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
 861
 862 #ifdef CAM_BOOT_DELAY
 863         /*
 864          * Override this value at compile time to assist our users
 865          * who don't use loader to boot a kernel.
 866          */
 867         xsoftc.boot_delay = CAM_BOOT_DELAY;
 868 #endif
 869         /*
 870          * The xpt layer is, itself, the equivelent of a SIM.
 871          * Allow 16 ccbs in the ccb pool for it.  This should
 872          * give decent parallelism when we probe busses and
 873          * perform other XPT functions.
 874          */
 875         devq = cam_simq_alloc(16);
 876         xpt_sim = cam_sim_alloc(xptaction,
 877                                 xptpoll,
 878                                 "xpt",
 879                                 /*softc*/NULL,
 880                                 /*unit*/0,
 881                                 /*mtx*/&xsoftc.xpt_lock,
 882                                 /*max_dev_transactions*/0,
 883                                 /*max_tagged_dev_transactions*/0,
 884                                 devq);
 885         if (xpt_sim == NULL)
 886                 return (ENOMEM);
 887
 888         mtx_lock(&xsoftc.xpt_lock);
 889         if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
 890                 mtx_unlock(&xsoftc.xpt_lock);
 891                 printf("xpt_init: xpt_bus_register failed with status %#x,"
 892                        " failing attach\n", status);
 893                 return (EINVAL);
 894         }
 895         mtx_unlock(&xsoftc.xpt_lock);
 896
 897         /*
 898          * Looking at the XPT from the SIM layer, the XPT is
 899          * the equivelent of a peripheral driver.  Allocate
 900          * a peripheral driver entry for us.
 901          */
 902         if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
 903                                       CAM_TARGET_WILDCARD,
 904                                       CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
 905                 printf("xpt_init: xpt_create_path failed with status %#x,"
 906                        " failing attach\n", status);
 907                 return (EINVAL);
 908         }
 909         xpt_path_lock(path);
 910         cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
 911                          path, NULL, 0, xpt_sim);
 912         xpt_path_unlock(path);
 913         xpt_free_path(path);
 914
 915         if (cam_num_doneqs < 1)
 916                 cam_num_doneqs = 1 + mp_ncpus / 6;
 917         else if (cam_num_doneqs > MAXCPU)
 918                 cam_num_doneqs = MAXCPU;
 919         for (i = 0; i < cam_num_doneqs; i++) {
 920                 mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
 921                     MTX_DEF);
 922                 STAILQ_INIT(&cam_doneqs[i].cam_doneq);
 923                 error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
 924                     &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
 925                 if (error != 0) {
 926                         cam_num_doneqs = i;
 927                         break;
 928                 }
 929         }
 930         if (cam_num_doneqs < 1) {
 931                 printf("xpt_init: Cannot init completion queues "
 932                        "- failing attach\n");
 933                 return (ENOMEM);
 934         }
 935         /*
 936          * Register a callback for when interrupts are enabled.
 937          */
 938         xsoftc.xpt_config_hook =
 939             (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
 940                                               M_CAMXPT, M_NOWAIT | M_ZERO);
 941         if (xsoftc.xpt_config_hook == NULL) {
 942                 printf("xpt_init: Cannot malloc config hook "
 943                        "- failing attach\n");
 944                 return (ENOMEM);
 945         }
 946         xsoftc.xpt_config_hook->ich_func = xpt_config;
 947         if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
 948                 free (xsoftc.xpt_config_hook, M_CAMXPT);
 949                 printf("xpt_init: config_intrhook_establish failed "
 950                        "- failing attach\n");
 951         }
 952
 953         return (0);
 954 }
 955
 956 static cam_status
 957 xptregister(struct cam_periph *periph, void *arg)
 958 {
 959         struct cam_sim *xpt_sim;
 960
 961         if (periph == NULL) {
 962                 printf("xptregister: periph was NULL!!\n");
 963                 return(CAM_REQ_CMP_ERR);
 964         }
 965
 966         xpt_sim = (struct cam_sim *)arg;
 967         xpt_sim->softc = periph;
 968         xpt_periph = periph;
 969         periph->softc = NULL;
 970
 971         return(CAM_REQ_CMP);
 972 }
 973
 974 int32_t
 975 xpt_add_periph(struct cam_periph *periph)
 976 {
 977         struct cam_ed *device;
 978         int32_t  status;
 979
 980         TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
 981         device = periph->path->device;
 982         status = CAM_REQ_CMP;
 983         if (device != NULL) {
 984                 mtx_lock(&device->target->bus->eb_mtx);
 985                 device->generation++;
 986                 SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
 987                 mtx_unlock(&device->target->bus->eb_mtx);
 988                 atomic_add_32(&xsoftc.xpt_generation, 1);
 989         }
 990
 991         return (status);
 992 }
 993
 994 void
 995 xpt_remove_periph(struct cam_periph *periph)
 996 {
 997         struct cam_ed *device;
 998
 999         device = periph->path->device;
1000         if (device != NULL) {
1001                 mtx_lock(&device->target->bus->eb_mtx);
1002                 device->generation++;
1003                 SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1004                 mtx_unlock(&device->target->bus->eb_mtx);
1005                 atomic_add_32(&xsoftc.xpt_generation, 1);
1006         }
1007 }
1008
1009
1010 void
1011 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1012 {
1013         struct  cam_path *path = periph->path;
1014
1015         cam_periph_assert(periph, MA_OWNED);
1016         periph->flags |= CAM_PERIPH_ANNOUNCED;
1017
1018         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1019                periph->periph_name, periph->unit_number,
1020                path->bus->sim->sim_name,
1021                path->bus->sim->unit_number,
1022                path->bus->sim->bus_id,
1023                path->bus->path_id,
1024                path->target->target_id,
1025                (uintmax_t)path->device->lun_id);
1026         printf("%s%d: ", periph->periph_name, periph->unit_number);
1027         if (path->device->protocol == PROTO_SCSI)
1028                 scsi_print_inquiry(&path->device->inq_data);
1029         else if (path->device->protocol == PROTO_ATA ||
1030             path->device->protocol == PROTO_SATAPM)
1031                 ata_print_ident(&path->device->ident_data);
1032         else if (path->device->protocol == PROTO_SEMB)
1033                 semb_print_ident(
1034                     (struct sep_identify_data *)&path->device->ident_data);
1035         else
1036                 printf("Unknown protocol device\n");
1037         if (path->device->serial_num_len > 0) {
1038                 /* Don't wrap the screen  - print only the first 60 chars */
1039                 printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1040                        periph->unit_number, path->device->serial_num);
1041         }
1042         /* Announce transport details. */
1043         (*(path->bus->xport->announce))(periph);
1044         /* Announce command queueing. */
1045         if (path->device->inq_flags & SID_CmdQue
1046          || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1047                 printf("%s%d: Command Queueing enabled\n",
1048                        periph->periph_name, periph->unit_number);
1049         }
1050         /* Announce caller's details if they've passed in. */
1051         if (announce_string != NULL)
1052                 printf("%s%d: %s\n", periph->periph_name,
1053                        periph->unit_number, announce_string);
1054 }
1055
1056 void
1057 xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1058 {
1059         if (quirks != 0) {
1060                 printf("%s%d: quirks=0x%b\n", periph->periph_name,
1061                     periph->unit_number, quirks, bit_string);
1062         }
1063 }
1064
1065 void
1066 xpt_denounce_periph(struct cam_periph *periph)
1067 {
1068         struct  cam_path *path = periph->path;
1069
1070         cam_periph_assert(periph, MA_OWNED);
1071         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1072                periph->periph_name, periph->unit_number,
1073                path->bus->sim->sim_name,
1074                path->bus->sim->unit_number,
1075                path->bus->sim->bus_id,
1076                path->bus->path_id,
1077                path->target->target_id,
1078                (uintmax_t)path->device->lun_id);
1079         printf("%s%d: ", periph->periph_name, periph->unit_number);
1080         if (path->device->protocol == PROTO_SCSI)
1081                 scsi_print_inquiry_short(&path->device->inq_data);
1082         else if (path->device->protocol == PROTO_ATA ||
1083             path->device->protocol == PROTO_SATAPM)
1084                 ata_print_ident_short(&path->device->ident_data);
1085         else if (path->device->protocol == PROTO_SEMB)
1086                 semb_print_ident_short(
1087                     (struct sep_identify_data *)&path->device->ident_data);
1088         else
1089                 printf("Unknown protocol device");
1090         if (path->device->serial_num_len > 0)
1091                 printf(" s/n %.60s", path->device->serial_num);
1092         printf(" detached\n");
1093 }
1094
1095
1096 int
1097 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1098 {
1099         int ret = -1, l;
1100         struct ccb_dev_advinfo cdai;
1101         struct scsi_vpd_id_descriptor *idd;
1102
1103         xpt_path_assert(path, MA_OWNED);
1104
1105         memset(&cdai, 0, sizeof(cdai));
1106         xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1107         cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1108         cdai.bufsiz = len;
1109
1110         if (!strcmp(attr, "GEOM::ident"))
1111                 cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1112         else if (!strcmp(attr, "GEOM::physpath"))
1113                 cdai.buftype = CDAI_TYPE_PHYS_PATH;
1114         else if (strcmp(attr, "GEOM::lunid") == 0 ||
1115                  strcmp(attr, "GEOM::lunname") == 0) {
1116                 cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1117                 cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1118         } else
1119                 goto out;
1120
1121         cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1122         if (cdai.buf == NULL) {
1123                 ret = ENOMEM;
1124                 goto out;
1125         }
1126         xpt_action((union ccb *)&cdai); /* can only be synchronous */
1127         if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1128                 cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1129         if (cdai.provsiz == 0)
1130                 goto out;
1131         if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1132                 if (strcmp(attr, "GEOM::lunid") == 0) {
1133                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1134                             cdai.provsiz, scsi_devid_is_lun_naa);
1135                         if (idd == NULL)
1136                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1137                                     cdai.provsiz, scsi_devid_is_lun_eui64);
1138                 } else
1139                         idd = NULL;
1140                 if (idd == NULL)
1141                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1142                             cdai.provsiz, scsi_devid_is_lun_t10);
1143                 if (idd == NULL)
1144                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1145                             cdai.provsiz, scsi_devid_is_lun_name);
1146                 if (idd == NULL)
1147                         goto out;
1148                 ret = 0;
1149                 if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1150                         if (idd->length < len) {
1151                                 for (l = 0; l < idd->length; l++)
1152                                         buf[l] = idd->identifier[l] ?
1153                                             idd->identifier[l] : ' ';
1154                                 buf[l] = 0;
1155                         } else
1156                                 ret = EFAULT;
1157                 } else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1158                         l = strnlen(idd->identifier, idd->length);
1159                         if (l < len) {
1160                                 bcopy(idd->identifier, buf, l);
1161                                 buf[l] = 0;
1162                         } else
1163                                 ret = EFAULT;
1164                 } else {
1165                         if (idd->length * 2 < len) {
1166                                 for (l = 0; l < idd->length; l++)
1167                                         sprintf(buf + l * 2, "%02x",
1168                                             idd->identifier[l]);
1169                         } else
1170                                 ret = EFAULT;
1171                 }
1172         } else {
1173                 ret = 0;
1174                 if (strlcpy(buf, cdai.buf, len) >= len)
1175                         ret = EFAULT;
1176         }
1177
1178 out:
1179         if (cdai.buf != NULL)
1180                 free(cdai.buf, M_CAMXPT);
1181         return ret;
1182 }
1183
1184 static dev_match_ret
1185 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1186             struct cam_eb *bus)
1187 {
1188         dev_match_ret retval;
1189         int i;
1190
1191         retval = DM_RET_NONE;
1192
1193         /*
1194          * If we aren't given something to match against, that's an error.
1195          */
1196         if (bus == NULL)
1197                 return(DM_RET_ERROR);
1198
1199         /*
1200          * If there are no match entries, then this bus matches no
1201          * matter what.
1202          */
1203         if ((patterns == NULL) || (num_patterns == 0))
1204                 return(DM_RET_DESCEND | DM_RET_COPY);
1205
1206         for (i = 0; i < num_patterns; i++) {
1207                 struct bus_match_pattern *cur_pattern;
1208
1209                 /*
1210                  * If the pattern in question isn't for a bus node, we
1211                  * aren't interested.  However, we do indicate to the
1212                  * calling routine that we should continue descending the
1213                  * tree, since the user wants to match against lower-level
1214                  * EDT elements.
1215                  */
1216                 if (patterns[i].type != DEV_MATCH_BUS) {
1217                         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1218                                 retval |= DM_RET_DESCEND;
1219                         continue;
1220                 }
1221
1222                 cur_pattern = &patterns[i].pattern.bus_pattern;
1223
1224                 /*
1225                  * If they want to match any bus node, we give them any
1226                  * device node.
1227                  */
1228                 if (cur_pattern->flags == BUS_MATCH_ANY) {
1229                         /* set the copy flag */
1230                         retval |= DM_RET_COPY;
1231
1232                         /*
1233                          * If we've already decided on an action, go ahead
1234                          * and return.
1235                          */
1236                         if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1237                                 return(retval);
1238                 }
1239
1240                 /*
1241                  * Not sure why someone would do this...
1242                  */
1243                 if (cur_pattern->flags == BUS_MATCH_NONE)
1244                         continue;
1245
1246                 if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1247                  && (cur_pattern->path_id != bus->path_id))
1248                         continue;
1249
1250                 if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1251                  && (cur_pattern->bus_id != bus->sim->bus_id))
1252                         continue;
1253
1254                 if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1255                  && (cur_pattern->unit_number != bus->sim->unit_number))
1256                         continue;
1257
1258                 if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1259                  && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1260                              DEV_IDLEN) != 0))
1261                         continue;
1262
1263                 /*
1264                  * If we get to this point, the user definitely wants
1265                  * information on this bus.  So tell the caller to copy the
1266                  * data out.
1267                  */
1268                 retval |= DM_RET_COPY;
1269
1270                 /*
1271                  * If the return action has been set to descend, then we
1272                  * know that we've already seen a non-bus matching
1273                  * expression, therefore we need to further descend the tree.
1274                  * This won't change by continuing around the loop, so we
1275                  * go ahead and return.  If we haven't seen a non-bus
1276                  * matching expression, we keep going around the loop until
1277                  * we exhaust the matching expressions.  We'll set the stop
1278                  * flag once we fall out of the loop.
1279                  */
1280                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1281                         return(retval);
1282         }
1283
1284         /*
1285          * If the return action hasn't been set to descend yet, that means
1286          * we haven't seen anything other than bus matching patterns.  So
1287          * tell the caller to stop descending the tree -- the user doesn't
1288          * want to match against lower level tree elements.
1289          */
1290         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1291                 retval |= DM_RET_STOP;
1292
1293         return(retval);
1294 }
1295
1296 static dev_match_ret
1297 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1298                struct cam_ed *device)
1299 {
1300         dev_match_ret retval;
1301         int i;
1302
1303         retval = DM_RET_NONE;
1304
1305         /*
1306          * If we aren't given something to match against, that's an error.
1307          */
1308         if (device == NULL)
1309                 return(DM_RET_ERROR);
1310
1311         /*
1312          * If there are no match entries, then this device matches no
1313          * matter what.
1314          */
1315         if ((patterns == NULL) || (num_patterns == 0))
1316                 return(DM_RET_DESCEND | DM_RET_COPY);
1317
1318         for (i = 0; i < num_patterns; i++) {
1319                 struct device_match_pattern *cur_pattern;
1320                 struct scsi_vpd_device_id *device_id_page;
1321
1322                 /*
1323                  * If the pattern in question isn't for a device node, we
1324                  * aren't interested.
1325                  */
1326                 if (patterns[i].type != DEV_MATCH_DEVICE) {
1327                         if ((patterns[i].type == DEV_MATCH_PERIPH)
1328                          && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1329                                 retval |= DM_RET_DESCEND;
1330                         continue;
1331                 }
1332
1333                 cur_pattern = &patterns[i].pattern.device_pattern;
1334
1335                 /* Error out if mutually exclusive options are specified. */
1336                 if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1337                  == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1338                         return(DM_RET_ERROR);
1339
1340                 /*
1341                  * If they want to match any device node, we give them any
1342                  * device node.
1343                  */
1344                 if (cur_pattern->flags == DEV_MATCH_ANY)
1345                         goto copy_dev_node;
1346
1347                 /*
1348                  * Not sure why someone would do this...
1349                  */
1350                 if (cur_pattern->flags == DEV_MATCH_NONE)
1351                         continue;
1352
1353                 if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1354                  && (cur_pattern->path_id != device->target->bus->path_id))
1355                         continue;
1356
1357                 if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1358                  && (cur_pattern->target_id != device->target->target_id))
1359                         continue;
1360
1361                 if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1362                  && (cur_pattern->target_lun != device->lun_id))
1363                         continue;
1364
1365                 if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1366                  && (cam_quirkmatch((caddr_t)&device->inq_data,
1367                                     (caddr_t)&cur_pattern->data.inq_pat,
1368                                     1, sizeof(cur_pattern->data.inq_pat),
1369                                     scsi_static_inquiry_match) == NULL))
1370                         continue;
1371
1372                 device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1373                 if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1374                  && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1375                   || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1376                                       device->device_id_len
1377                                     - SVPD_DEVICE_ID_HDR_LEN,
1378                                       cur_pattern->data.devid_pat.id,
1379                                       cur_pattern->data.devid_pat.id_len) != 0))
1380                         continue;
1381
1382 copy_dev_node:
1383                 /*
1384                  * If we get to this point, the user definitely wants
1385                  * information on this device.  So tell the caller to copy
1386                  * the data out.
1387                  */
1388                 retval |= DM_RET_COPY;
1389
1390                 /*
1391                  * If the return action has been set to descend, then we
1392                  * know that we've already seen a peripheral matching
1393                  * expression, therefore we need to further descend the tree.
1394                  * This won't change by continuing around the loop, so we
1395                  * go ahead and return.  If we haven't seen a peripheral
1396                  * matching expression, we keep going around the loop until
1397                  * we exhaust the matching expressions.  We'll set the stop
1398                  * flag once we fall out of the loop.
1399                  */
1400                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1401                         return(retval);
1402         }
1403
1404         /*
1405          * If the return action hasn't been set to descend yet, that means
1406          * we haven't seen any peripheral matching patterns.  So tell the
1407          * caller to stop descending the tree -- the user doesn't want to
1408          * match against lower level tree elements.
1409          */
1410         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1411                 retval |= DM_RET_STOP;
1412
1413         return(retval);
1414 }
1415
1416 /*
1417  * Match a single peripheral against any number of match patterns.
1418  */
1419 static dev_match_ret
1420 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1421                struct cam_periph *periph)
1422 {
1423         dev_match_ret retval;
1424         int i;
1425
1426         /*
1427          * If we aren't given something to match against, that's an error.
1428          */
1429         if (periph == NULL)
1430                 return(DM_RET_ERROR);
1431
1432         /*
1433          * If there are no match entries, then this peripheral matches no
1434          * matter what.
1435          */
1436         if ((patterns == NULL) || (num_patterns == 0))
1437                 return(DM_RET_STOP | DM_RET_COPY);
1438
1439         /*
1440          * There aren't any nodes below a peripheral node, so there's no
1441          * reason to descend the tree any further.
1442          */
1443         retval = DM_RET_STOP;
1444
1445         for (i = 0; i < num_patterns; i++) {
1446                 struct periph_match_pattern *cur_pattern;
1447
1448                 /*
1449                  * If the pattern in question isn't for a peripheral, we
1450                  * aren't interested.
1451                  */
1452                 if (patterns[i].type != DEV_MATCH_PERIPH)
1453                         continue;
1454
1455                 cur_pattern = &patterns[i].pattern.periph_pattern;
1456
1457                 /*
1458                  * If they want to match on anything, then we will do so.
1459                  */
1460                 if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1461                         /* set the copy flag */
1462                         retval |= DM_RET_COPY;
1463
1464                         /*
1465                          * We've already set the return action to stop,
1466                          * since there are no nodes below peripherals in
1467                          * the tree.
1468                          */
1469                         return(retval);
1470                 }
1471
1472                 /*
1473                  * Not sure why someone would do this...
1474                  */
1475                 if (cur_pattern->flags == PERIPH_MATCH_NONE)
1476                         continue;
1477
1478                 if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1479                  && (cur_pattern->path_id != periph->path->bus->path_id))
1480                         continue;
1481
1482                 /*
1483                  * For the target and lun id's, we have to make sure the
1484                  * target and lun pointers aren't NULL.  The xpt peripheral
1485                  * has a wildcard target and device.
1486                  */
1487                 if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1488                  && ((periph->path->target == NULL)
1489                  ||(cur_pattern->target_id != periph->path->target->target_id)))
1490                         continue;
1491
1492                 if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1493                  && ((periph->path->device == NULL)
1494                  || (cur_pattern->target_lun != periph->path->device->lun_id)))
1495                         continue;
1496
1497                 if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1498                  && (cur_pattern->unit_number != periph->unit_number))
1499                         continue;
1500
1501                 if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1502                  && (strncmp(cur_pattern->periph_name, periph->periph_name,
1503                              DEV_IDLEN) != 0))
1504                         continue;
1505
1506                 /*
1507                  * If we get to this point, the user definitely wants
1508                  * information on this peripheral.  So tell the caller to
1509                  * copy the data out.
1510                  */
1511                 retval |= DM_RET_COPY;
1512
1513                 /*
1514                  * The return action has already been set to stop, since
1515                  * peripherals don't have any nodes below them in the EDT.
1516                  */
1517                 return(retval);
1518         }
1519
1520         /*
1521          * If we get to this point, the peripheral that was passed in
1522          * doesn't match any of the patterns.
1523          */
1524         return(retval);
1525 }
1526
1527 static int
1528 xptedtbusfunc(struct cam_eb *bus, void *arg)
1529 {
1530         struct ccb_dev_match *cdm;
1531         struct cam_et *target;
1532         dev_match_ret retval;
1533
1534         cdm = (struct ccb_dev_match *)arg;
1535
1536         /*
1537          * If our position is for something deeper in the tree, that means
1538          * that we've already seen this node.  So, we keep going down.
1539          */
1540         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1541          && (cdm->pos.cookie.bus == bus)
1542          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1543          && (cdm->pos.cookie.target != NULL))
1544                 retval = DM_RET_DESCEND;
1545         else
1546                 retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1547
1548         /*
1549          * If we got an error, bail out of the search.
1550          */
1551         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1552                 cdm->status = CAM_DEV_MATCH_ERROR;
1553                 return(0);
1554         }
1555
1556         /*
1557          * If the copy flag is set, copy this bus out.
1558          */
1559         if (retval & DM_RET_COPY) {
1560                 int spaceleft, j;
1561
1562                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1563                         sizeof(struct dev_match_result));
1564
1565                 /*
1566                  * If we don't have enough space to put in another
1567                  * match result, save our position and tell the
1568                  * user there are more devices to check.
1569                  */
1570                 if (spaceleft < sizeof(struct dev_match_result)) {
1571                         bzero(&cdm->pos, sizeof(cdm->pos));
1572                         cdm->pos.position_type =
1573                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1574
1575                         cdm->pos.cookie.bus = bus;
1576                         cdm->pos.generations[CAM_BUS_GENERATION]=
1577                                 xsoftc.bus_generation;
1578                         cdm->status = CAM_DEV_MATCH_MORE;
1579                         return(0);
1580                 }
1581                 j = cdm->num_matches;
1582                 cdm->num_matches++;
1583                 cdm->matches[j].type = DEV_MATCH_BUS;
1584                 cdm->matches[j].result.bus_result.path_id = bus->path_id;
1585                 cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1586                 cdm->matches[j].result.bus_result.unit_number =
1587                         bus->sim->unit_number;
1588                 strncpy(cdm->matches[j].result.bus_result.dev_name,
1589                         bus->sim->sim_name, DEV_IDLEN);
1590         }
1591
1592         /*
1593          * If the user is only interested in busses, there's no
1594          * reason to descend to the next level in the tree.
1595          */
1596         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1597                 return(1);
1598
1599         /*
1600          * If there is a target generation recorded, check it to
1601          * make sure the target list hasn't changed.
1602          */
1603         mtx_lock(&bus->eb_mtx);
1604         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1605          && (cdm->pos.cookie.bus == bus)
1606          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1607          && (cdm->pos.cookie.target != NULL)) {
1608                 if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1609                     bus->generation)) {
1610                         mtx_unlock(&bus->eb_mtx);
1611                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1612                         return (0);
1613                 }
1614                 target = (struct cam_et *)cdm->pos.cookie.target;
1615                 target->refcount++;
1616         } else
1617                 target = NULL;
1618         mtx_unlock(&bus->eb_mtx);
1619
1620         return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1621 }
1622
1623 static int
1624 xptedttargetfunc(struct cam_et *target, void *arg)
1625 {
1626         struct ccb_dev_match *cdm;
1627         struct cam_eb *bus;
1628         struct cam_ed *device;
1629
1630         cdm = (struct ccb_dev_match *)arg;
1631         bus = target->bus;
1632
1633         /*
1634          * If there is a device list generation recorded, check it to
1635          * make sure the device list hasn't changed.
1636          */
1637         mtx_lock(&bus->eb_mtx);
1638         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1639          && (cdm->pos.cookie.bus == bus)
1640          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1641          && (cdm->pos.cookie.target == target)
1642          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1643          && (cdm->pos.cookie.device != NULL)) {
1644                 if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1645                     target->generation) {
1646                         mtx_unlock(&bus->eb_mtx);
1647                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1648                         return(0);
1649                 }
1650                 device = (struct cam_ed *)cdm->pos.cookie.device;
1651                 device->refcount++;
1652         } else
1653                 device = NULL;
1654         mtx_unlock(&bus->eb_mtx);
1655
1656         return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1657 }
1658
1659 static int
1660 xptedtdevicefunc(struct cam_ed *device, void *arg)
1661 {
1662         struct cam_eb *bus;
1663         struct cam_periph *periph;
1664         struct ccb_dev_match *cdm;
1665         dev_match_ret retval;
1666
1667         cdm = (struct ccb_dev_match *)arg;
1668         bus = device->target->bus;
1669
1670         /*
1671          * If our position is for something deeper in the tree, that means
1672          * that we've already seen this node.  So, we keep going down.
1673          */
1674         if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1675          && (cdm->pos.cookie.device == device)
1676          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1677          && (cdm->pos.cookie.periph != NULL))
1678                 retval = DM_RET_DESCEND;
1679         else
1680                 retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1681                                         device);
1682
1683         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1684                 cdm->status = CAM_DEV_MATCH_ERROR;
1685                 return(0);
1686         }
1687
1688         /*
1689          * If the copy flag is set, copy this device out.
1690          */
1691         if (retval & DM_RET_COPY) {
1692                 int spaceleft, j;
1693
1694                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1695                         sizeof(struct dev_match_result));
1696
1697                 /*
1698                  * If we don't have enough space to put in another
1699                  * match result, save our position and tell the
1700                  * user there are more devices to check.
1701                  */
1702                 if (spaceleft < sizeof(struct dev_match_result)) {
1703                         bzero(&cdm->pos, sizeof(cdm->pos));
1704                         cdm->pos.position_type =
1705                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1706                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1707
1708                         cdm->pos.cookie.bus = device->target->bus;
1709                         cdm->pos.generations[CAM_BUS_GENERATION]=
1710                                 xsoftc.bus_generation;
1711                         cdm->pos.cookie.target = device->target;
1712                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1713                                 device->target->bus->generation;
1714                         cdm->pos.cookie.device = device;
1715                         cdm->pos.generations[CAM_DEV_GENERATION] =
1716                                 device->target->generation;
1717                         cdm->status = CAM_DEV_MATCH_MORE;
1718                         return(0);
1719                 }
1720                 j = cdm->num_matches;
1721                 cdm->num_matches++;
1722                 cdm->matches[j].type = DEV_MATCH_DEVICE;
1723                 cdm->matches[j].result.device_result.path_id =
1724                         device->target->bus->path_id;
1725                 cdm->matches[j].result.device_result.target_id =
1726                         device->target->target_id;
1727                 cdm->matches[j].result.device_result.target_lun =
1728                         device->lun_id;
1729                 cdm->matches[j].result.device_result.protocol =
1730                         device->protocol;
1731                 bcopy(&device->inq_data,
1732                       &cdm->matches[j].result.device_result.inq_data,
1733                       sizeof(struct scsi_inquiry_data));
1734                 bcopy(&device->ident_data,
1735                       &cdm->matches[j].result.device_result.ident_data,
1736                       sizeof(struct ata_params));
1737
1738                 /* Let the user know whether this device is unconfigured */
1739                 if (device->flags & CAM_DEV_UNCONFIGURED)
1740                         cdm->matches[j].result.device_result.flags =
1741                                 DEV_RESULT_UNCONFIGURED;
1742                 else
1743                         cdm->matches[j].result.device_result.flags =
1744                                 DEV_RESULT_NOFLAG;
1745         }
1746
1747         /*
1748          * If the user isn't interested in peripherals, don't descend
1749          * the tree any further.
1750          */
1751         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1752                 return(1);
1753
1754         /*
1755          * If there is a peripheral list generation recorded, make sure
1756          * it hasn't changed.
1757          */
1758         xpt_lock_buses();
1759         mtx_lock(&bus->eb_mtx);
1760         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1761          && (cdm->pos.cookie.bus == bus)
1762          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1763          && (cdm->pos.cookie.target == device->target)
1764          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1765          && (cdm->pos.cookie.device == device)
1766          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1767          && (cdm->pos.cookie.periph != NULL)) {
1768                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1769                     device->generation) {
1770                         mtx_unlock(&bus->eb_mtx);
1771                         xpt_unlock_buses();
1772                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1773                         return(0);
1774                 }
1775                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1776                 periph->refcount++;
1777         } else
1778                 periph = NULL;
1779         mtx_unlock(&bus->eb_mtx);
1780         xpt_unlock_buses();
1781
1782         return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1783 }
1784
1785 static int
1786 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1787 {
1788         struct ccb_dev_match *cdm;
1789         dev_match_ret retval;
1790
1791         cdm = (struct ccb_dev_match *)arg;
1792
1793         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1794
1795         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1796                 cdm->status = CAM_DEV_MATCH_ERROR;
1797                 return(0);
1798         }
1799
1800         /*
1801          * If the copy flag is set, copy this peripheral out.
1802          */
1803         if (retval & DM_RET_COPY) {
1804                 int spaceleft, j;
1805
1806                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1807                         sizeof(struct dev_match_result));
1808
1809                 /*
1810                  * If we don't have enough space to put in another
1811                  * match result, save our position and tell the
1812                  * user there are more devices to check.
1813                  */
1814                 if (spaceleft < sizeof(struct dev_match_result)) {
1815                         bzero(&cdm->pos, sizeof(cdm->pos));
1816                         cdm->pos.position_type =
1817                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1818                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1819                                 CAM_DEV_POS_PERIPH;
1820
1821                         cdm->pos.cookie.bus = periph->path->bus;
1822                         cdm->pos.generations[CAM_BUS_GENERATION]=
1823                                 xsoftc.bus_generation;
1824                         cdm->pos.cookie.target = periph->path->target;
1825                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1826                                 periph->path->bus->generation;
1827                         cdm->pos.cookie.device = periph->path->device;
1828                         cdm->pos.generations[CAM_DEV_GENERATION] =
1829                                 periph->path->target->generation;
1830                         cdm->pos.cookie.periph = periph;
1831                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
1832                                 periph->path->device->generation;
1833                         cdm->status = CAM_DEV_MATCH_MORE;
1834                         return(0);
1835                 }
1836
1837                 j = cdm->num_matches;
1838                 cdm->num_matches++;
1839                 cdm->matches[j].type = DEV_MATCH_PERIPH;
1840                 cdm->matches[j].result.periph_result.path_id =
1841                         periph->path->bus->path_id;
1842                 cdm->matches[j].result.periph_result.target_id =
1843                         periph->path->target->target_id;
1844                 cdm->matches[j].result.periph_result.target_lun =
1845                         periph->path->device->lun_id;
1846                 cdm->matches[j].result.periph_result.unit_number =
1847                         periph->unit_number;
1848                 strncpy(cdm->matches[j].result.periph_result.periph_name,
1849                         periph->periph_name, DEV_IDLEN);
1850         }
1851
1852         return(1);
1853 }
1854
1855 static int
1856 xptedtmatch(struct ccb_dev_match *cdm)
1857 {
1858         struct cam_eb *bus;
1859         int ret;
1860
1861         cdm->num_matches = 0;
1862
1863         /*
1864          * Check the bus list generation.  If it has changed, the user
1865          * needs to reset everything and start over.
1866          */
1867         xpt_lock_buses();
1868         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1869          && (cdm->pos.cookie.bus != NULL)) {
1870                 if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1871                     xsoftc.bus_generation) {
1872                         xpt_unlock_buses();
1873                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1874                         return(0);
1875                 }
1876                 bus = (struct cam_eb *)cdm->pos.cookie.bus;
1877                 bus->refcount++;
1878         } else
1879                 bus = NULL;
1880         xpt_unlock_buses();
1881
1882         ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1883
1884         /*
1885          * If we get back 0, that means that we had to stop before fully
1886          * traversing the EDT.  It also means that one of the subroutines
1887          * has set the status field to the proper value.  If we get back 1,
1888          * we've fully traversed the EDT and copied out any matching entries.
1889          */
1890         if (ret == 1)
1891                 cdm->status = CAM_DEV_MATCH_LAST;
1892
1893         return(ret);
1894 }
1895
1896 static int
1897 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1898 {
1899         struct cam_periph *periph;
1900         struct ccb_dev_match *cdm;
1901
1902         cdm = (struct ccb_dev_match *)arg;
1903
1904         xpt_lock_buses();
1905         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1906          && (cdm->pos.cookie.pdrv == pdrv)
1907          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1908          && (cdm->pos.cookie.periph != NULL)) {
1909                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1910                     (*pdrv)->generation) {
1911                         xpt_unlock_buses();
1912                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1913                         return(0);
1914                 }
1915                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1916                 periph->refcount++;
1917         } else
1918                 periph = NULL;
1919         xpt_unlock_buses();
1920
1921         return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1922 }
1923
1924 static int
1925 xptplistperiphfunc(struct cam_periph *periph, void *arg)
1926 {
1927         struct ccb_dev_match *cdm;
1928         dev_match_ret retval;
1929
1930         cdm = (struct ccb_dev_match *)arg;
1931
1932         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1933
1934         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1935                 cdm->status = CAM_DEV_MATCH_ERROR;
1936                 return(0);
1937         }
1938
1939         /*
1940          * If the copy flag is set, copy this peripheral out.
1941          */
1942         if (retval & DM_RET_COPY) {
1943                 int spaceleft, j;
1944
1945                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1946                         sizeof(struct dev_match_result));
1947
1948                 /*
1949                  * If we don't have enough space to put in another
1950                  * match result, save our position and tell the
1951                  * user there are more devices to check.
1952                  */
1953                 if (spaceleft < sizeof(struct dev_match_result)) {
1954                         struct periph_driver **pdrv;
1955
1956                         pdrv = NULL;
1957                         bzero(&cdm->pos, sizeof(cdm->pos));
1958                         cdm->pos.position_type =
1959                                 CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1960                                 CAM_DEV_POS_PERIPH;
1961
1962                         /*
1963                          * This may look a bit non-sensical, but it is
1964                          * actually quite logical.  There are very few
1965                          * peripheral drivers, and bloating every peripheral
1966                          * structure with a pointer back to its parent
1967                          * peripheral driver linker set entry would cost
1968                          * more in the long run than doing this quick lookup.
1969                          */
1970                         for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1971                                 if (strcmp((*pdrv)->driver_name,
1972                                     periph->periph_name) == 0)
1973                                         break;
1974                         }
1975
1976                         if (*pdrv == NULL) {
1977                                 cdm->status = CAM_DEV_MATCH_ERROR;
1978                                 return(0);
1979                         }
1980
1981                         cdm->pos.cookie.pdrv = pdrv;
1982                         /*
1983                          * The periph generation slot does double duty, as
1984                          * does the periph pointer slot.  They are used for
1985                          * both edt and pdrv lookups and positioning.
1986                          */
1987                         cdm->pos.cookie.periph = periph;
1988                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
1989                                 (*pdrv)->generation;
1990                         cdm->status = CAM_DEV_MATCH_MORE;
1991                         return(0);
1992                 }
1993
1994                 j = cdm->num_matches;
1995                 cdm->num_matches++;
1996                 cdm->matches[j].type = DEV_MATCH_PERIPH;
1997                 cdm->matches[j].result.periph_result.path_id =
1998                         periph->path->bus->path_id;
1999
2000                 /*
2001                  * The transport layer peripheral doesn't have a target or
2002                  * lun.
2003                  */
2004                 if (periph->path->target)
2005                         cdm->matches[j].result.periph_result.target_id =
2006                                 periph->path->target->target_id;
2007                 else
2008                         cdm->matches[j].result.periph_result.target_id =
2009                                 CAM_TARGET_WILDCARD;
2010
2011                 if (periph->path->device)
2012                         cdm->matches[j].result.periph_result.target_lun =
2013                                 periph->path->device->lun_id;
2014                 else
2015                         cdm->matches[j].result.periph_result.target_lun =
2016                                 CAM_LUN_WILDCARD;
2017
2018                 cdm->matches[j].result.periph_result.unit_number =
2019                         periph->unit_number;
2020                 strncpy(cdm->matches[j].result.periph_result.periph_name,
2021                         periph->periph_name, DEV_IDLEN);
2022         }
2023
2024         return(1);
2025 }
2026
2027 static int
2028 xptperiphlistmatch(struct ccb_dev_match *cdm)
2029 {
2030         int ret;
2031
2032         cdm->num_matches = 0;
2033
2034         /*
2035          * At this point in the edt traversal function, we check the bus
2036          * list generation to make sure that no busses have been added or
2037          * removed since the user last sent a XPT_DEV_MATCH ccb through.
2038          * For the peripheral driver list traversal function, however, we
2039          * don't have to worry about new peripheral driver types coming or
2040          * going; they're in a linker set, and therefore can't change
2041          * without a recompile.
2042          */
2043
2044         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2045          && (cdm->pos.cookie.pdrv != NULL))
2046                 ret = xptpdrvtraverse(
2047                                 (struct periph_driver **)cdm->pos.cookie.pdrv,
2048                                 xptplistpdrvfunc, cdm);
2049         else
2050                 ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2051
2052         /*
2053          * If we get back 0, that means that we had to stop before fully
2054          * traversing the peripheral driver tree.  It also means that one of
2055          * the subroutines has set the status field to the proper value.  If
2056          * we get back 1, we've fully traversed the EDT and copied out any
2057          * matching entries.
2058          */
2059         if (ret == 1)
2060                 cdm->status = CAM_DEV_MATCH_LAST;
2061
2062         return(ret);
2063 }
2064
2065 static int
2066 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2067 {
2068         struct cam_eb *bus, *next_bus;
2069         int retval;
2070
2071         retval = 1;
2072         if (start_bus)
2073                 bus = start_bus;
2074         else {
2075                 xpt_lock_buses();
2076                 bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2077                 if (bus == NULL) {
2078                         xpt_unlock_buses();
2079                         return (retval);
2080                 }
2081                 bus->refcount++;
2082                 xpt_unlock_buses();
2083         }
2084         for (; bus != NULL; bus = next_bus) {
2085                 retval = tr_func(bus, arg);
2086                 if (retval == 0) {
2087                         xpt_release_bus(bus);
2088                         break;
2089                 }
2090                 xpt_lock_buses();
2091                 next_bus = TAILQ_NEXT(bus, links);
2092                 if (next_bus)
2093                         next_bus->refcount++;
2094                 xpt_unlock_buses();
2095                 xpt_release_bus(bus);
2096         }
2097         return(retval);
2098 }
2099
2100 static int
2101 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2102                   xpt_targetfunc_t *tr_func, void *arg)
2103 {
2104         struct cam_et *target, *next_target;
2105         int retval;
2106
2107         retval = 1;
2108         if (start_target)
2109                 target = start_target;
2110         else {
2111                 mtx_lock(&bus->eb_mtx);
2112                 target = TAILQ_FIRST(&bus->et_entries);
2113                 if (target == NULL) {
2114                         mtx_unlock(&bus->eb_mtx);
2115                         return (retval);
2116                 }
2117                 target->refcount++;
2118                 mtx_unlock(&bus->eb_mtx);
2119         }
2120         for (; target != NULL; target = next_target) {
2121                 retval = tr_func(target, arg);
2122                 if (retval == 0) {
2123                         xpt_release_target(target);
2124                         break;
2125                 }
2126                 mtx_lock(&bus->eb_mtx);
2127                 next_target = TAILQ_NEXT(target, links);
2128                 if (next_target)
2129                         next_target->refcount++;
2130                 mtx_unlock(&bus->eb_mtx);
2131                 xpt_release_target(target);
2132         }
2133         return(retval);
2134 }
2135
2136 static int
2137 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2138                   xpt_devicefunc_t *tr_func, void *arg)
2139 {
2140         struct cam_eb *bus;
2141         struct cam_ed *device, *next_device;
2142         int retval;
2143
2144         retval = 1;
2145         bus = target->bus;
2146         if (start_device)
2147                 device = start_device;
2148         else {
2149                 mtx_lock(&bus->eb_mtx);
2150                 device = TAILQ_FIRST(&target->ed_entries);
2151                 if (device == NULL) {
2152                         mtx_unlock(&bus->eb_mtx);
2153                         return (retval);
2154                 }
2155                 device->refcount++;
2156                 mtx_unlock(&bus->eb_mtx);
2157         }
2158         for (; device != NULL; device = next_device) {
2159                 mtx_lock(&device->device_mtx);
2160                 retval = tr_func(device, arg);
2161                 mtx_unlock(&device->device_mtx);
2162                 if (retval == 0) {
2163                         xpt_release_device(device);
2164                         break;
2165                 }
2166                 mtx_lock(&bus->eb_mtx);
2167                 next_device = TAILQ_NEXT(device, links);
2168                 if (next_device)
2169                         next_device->refcount++;
2170                 mtx_unlock(&bus->eb_mtx);
2171                 xpt_release_device(device);
2172         }
2173         return(retval);
2174 }
2175
2176 static int
2177 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2178                   xpt_periphfunc_t *tr_func, void *arg)
2179 {
2180         struct cam_eb *bus;
2181         struct cam_periph *periph, *next_periph;
2182         int retval;
2183
2184         retval = 1;
2185
2186         bus = device->target->bus;
2187         if (start_periph)
2188                 periph = start_periph;
2189         else {
2190                 xpt_lock_buses();
2191                 mtx_lock(&bus->eb_mtx);
2192                 periph = SLIST_FIRST(&device->periphs);
2193                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2194                         periph = SLIST_NEXT(periph, periph_links);
2195                 if (periph == NULL) {
2196                         mtx_unlock(&bus->eb_mtx);
2197                         xpt_unlock_buses();
2198                         return (retval);
2199                 }
2200                 periph->refcount++;
2201                 mtx_unlock(&bus->eb_mtx);
2202                 xpt_unlock_buses();
2203         }
2204         for (; periph != NULL; periph = next_periph) {
2205                 retval = tr_func(periph, arg);
2206                 if (retval == 0) {
2207                         cam_periph_release_locked(periph);
2208                         break;
2209                 }
2210                 xpt_lock_buses();
2211                 mtx_lock(&bus->eb_mtx);
2212                 next_periph = SLIST_NEXT(periph, periph_links);
2213                 while (next_periph != NULL &&
2214                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2215                         next_periph = SLIST_NEXT(next_periph, periph_links);
2216                 if (next_periph)
2217                         next_periph->refcount++;
2218                 mtx_unlock(&bus->eb_mtx);
2219                 xpt_unlock_buses();
2220                 cam_periph_release_locked(periph);
2221         }
2222         return(retval);
2223 }
2224
2225 static int
2226 xptpdrvtraverse(struct periph_driver **start_pdrv,
2227                 xpt_pdrvfunc_t *tr_func, void *arg)
2228 {
2229         struct periph_driver **pdrv;
2230         int retval;
2231
2232         retval = 1;
2233
2234         /*
2235          * We don't traverse the peripheral driver list like we do the
2236          * other lists, because it is a linker set, and therefore cannot be
2237          * changed during runtime.  If the peripheral driver list is ever
2238          * re-done to be something other than a linker set (i.e. it can
2239          * change while the system is running), the list traversal should
2240          * be modified to work like the other traversal functions.
2241          */
2242         for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2243              *pdrv != NULL; pdrv++) {
2244                 retval = tr_func(pdrv, arg);
2245
2246                 if (retval == 0)
2247                         return(retval);
2248         }
2249
2250         return(retval);
2251 }
2252
2253 static int
2254 xptpdperiphtraverse(struct periph_driver **pdrv,
2255                     struct cam_periph *start_periph,
2256                     xpt_periphfunc_t *tr_func, void *arg)
2257 {
2258         struct cam_periph *periph, *next_periph;
2259         int retval;
2260
2261         retval = 1;
2262
2263         if (start_periph)
2264                 periph = start_periph;
2265         else {
2266                 xpt_lock_buses();
2267                 periph = TAILQ_FIRST(&(*pdrv)->units);
2268                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2269                         periph = TAILQ_NEXT(periph, unit_links);
2270                 if (periph == NULL) {
2271                         xpt_unlock_buses();
2272                         return (retval);
2273                 }
2274                 periph->refcount++;
2275                 xpt_unlock_buses();
2276         }
2277         for (; periph != NULL; periph = next_periph) {
2278                 cam_periph_lock(periph);
2279                 retval = tr_func(periph, arg);
2280                 cam_periph_unlock(periph);
2281                 if (retval == 0) {
2282                         cam_periph_release(periph);
2283                         break;
2284                 }
2285                 xpt_lock_buses();
2286                 next_periph = TAILQ_NEXT(periph, unit_links);
2287                 while (next_periph != NULL &&
2288                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2289                         next_periph = TAILQ_NEXT(next_periph, unit_links);
2290                 if (next_periph)
2291                         next_periph->refcount++;
2292                 xpt_unlock_buses();
2293                 cam_periph_release(periph);
2294         }
2295         return(retval);
2296 }
2297
2298 static int
2299 xptdefbusfunc(struct cam_eb *bus, void *arg)
2300 {
2301         struct xpt_traverse_config *tr_config;
2302
2303         tr_config = (struct xpt_traverse_config *)arg;
2304
2305         if (tr_config->depth == XPT_DEPTH_BUS) {
2306                 xpt_busfunc_t *tr_func;
2307
2308                 tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2309
2310                 return(tr_func(bus, tr_config->tr_arg));
2311         } else
2312                 return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2313 }
2314
2315 static int
2316 xptdeftargetfunc(struct cam_et *target, void *arg)
2317 {
2318         struct xpt_traverse_config *tr_config;
2319
2320         tr_config = (struct xpt_traverse_config *)arg;
2321
2322         if (tr_config->depth == XPT_DEPTH_TARGET) {
2323                 xpt_targetfunc_t *tr_func;
2324
2325                 tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2326
2327                 return(tr_func(target, tr_config->tr_arg));
2328         } else
2329                 return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2330 }
2331
2332 static int
2333 xptdefdevicefunc(struct cam_ed *device, void *arg)
2334 {
2335         struct xpt_traverse_config *tr_config;
2336
2337         tr_config = (struct xpt_traverse_config *)arg;
2338
2339         if (tr_config->depth == XPT_DEPTH_DEVICE) {
2340                 xpt_devicefunc_t *tr_func;
2341
2342                 tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2343
2344                 return(tr_func(device, tr_config->tr_arg));
2345         } else
2346                 return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2347 }
2348
2349 static int
2350 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2351 {
2352         struct xpt_traverse_config *tr_config;
2353         xpt_periphfunc_t *tr_func;
2354
2355         tr_config = (struct xpt_traverse_config *)arg;
2356
2357         tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2358
2359         /*
2360          * Unlike the other default functions, we don't check for depth
2361          * here.  The peripheral driver level is the last level in the EDT,
2362          * so if we're here, we should execute the function in question.
2363          */
2364         return(tr_func(periph, tr_config->tr_arg));
2365 }
2366
2367 /*
2368  * Execute the given function for every bus in the EDT.
2369  */
2370 static int
2371 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2372 {
2373         struct xpt_traverse_config tr_config;
2374
2375         tr_config.depth = XPT_DEPTH_BUS;
2376         tr_config.tr_func = tr_func;
2377         tr_config.tr_arg = arg;
2378
2379         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2380 }
2381
2382 /*
2383  * Execute the given function for every device in the EDT.
2384  */
2385 static int
2386 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2387 {
2388         struct xpt_traverse_config tr_config;
2389
2390         tr_config.depth = XPT_DEPTH_DEVICE;
2391         tr_config.tr_func = tr_func;
2392         tr_config.tr_arg = arg;
2393
2394         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2395 }
2396
2397 static int
2398 xptsetasyncfunc(struct cam_ed *device, void *arg)
2399 {
2400         struct cam_path path;
2401         struct ccb_getdev cgd;
2402         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2403
2404         /*
2405          * Don't report unconfigured devices (Wildcard devs,
2406          * devices only for target mode, device instances
2407          * that have been invalidated but are waiting for
2408          * their last reference count to be released).
2409          */
2410         if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2411                 return (1);
2412
2413         xpt_compile_path(&path,
2414                          NULL,
2415                          device->target->bus->path_id,
2416                          device->target->target_id,
2417                          device->lun_id);
2418         xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2419         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2420         xpt_action((union ccb *)&cgd);
2421         csa->callback(csa->callback_arg,
2422                             AC_FOUND_DEVICE,
2423                             &path, &cgd);
2424         xpt_release_path(&path);
2425
2426         return(1);
2427 }
2428
2429 static int
2430 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2431 {
2432         struct cam_path path;
2433         struct ccb_pathinq cpi;
2434         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2435
2436         xpt_compile_path(&path, /*periph*/NULL,
2437                          bus->path_id,
2438                          CAM_TARGET_WILDCARD,
2439                          CAM_LUN_WILDCARD);
2440         xpt_path_lock(&path);
2441         xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2442         cpi.ccb_h.func_code = XPT_PATH_INQ;
2443         xpt_action((union ccb *)&cpi);
2444         csa->callback(csa->callback_arg,
2445                             AC_PATH_REGISTERED,
2446                             &path, &cpi);
2447         xpt_path_unlock(&path);
2448         xpt_release_path(&path);
2449
2450         return(1);
2451 }
2452
2453 void
2454 xpt_action(union ccb *start_ccb)
2455 {
2456
2457         CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2458
2459         start_ccb->ccb_h.status = CAM_REQ_INPROG;
2460         (*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2461 }
2462
2463 void
2464 xpt_action_default(union ccb *start_ccb)
2465 {
2466         struct cam_path *path;
2467         struct cam_sim *sim;
2468         int lock;
2469
2470         path = start_ccb->ccb_h.path;
2471         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2472
2473         switch (start_ccb->ccb_h.func_code) {
2474         case XPT_SCSI_IO:
2475         {
2476                 struct cam_ed *device;
2477
2478                 /*
2479                  * For the sake of compatibility with SCSI-1
2480                  * devices that may not understand the identify
2481                  * message, we include lun information in the
2482                  * second byte of all commands.  SCSI-1 specifies
2483                  * that luns are a 3 bit value and reserves only 3
2484                  * bits for lun information in the CDB.  Later
2485                  * revisions of the SCSI spec allow for more than 8
2486                  * luns, but have deprecated lun information in the
2487                  * CDB.  So, if the lun won't fit, we must omit.
2488                  *
2489                  * Also be aware that during initial probing for devices,
2490                  * the inquiry information is unknown but initialized to 0.
2491                  * This means that this code will be exercised while probing
2492                  * devices with an ANSI revision greater than 2.
2493                  */
2494                 device = path->device;
2495                 if (device->protocol_version <= SCSI_REV_2
2496                  && start_ccb->ccb_h.target_lun < 8
2497                  && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2498
2499                         start_ccb->csio.cdb_io.cdb_bytes[1] |=
2500                             start_ccb->ccb_h.target_lun << 5;
2501                 }
2502                 start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2503         }
2504         /* FALLTHROUGH */
2505         case XPT_TARGET_IO:
2506         case XPT_CONT_TARGET_IO:
2507                 start_ccb->csio.sense_resid = 0;
2508                 start_ccb->csio.resid = 0;
2509                 /* FALLTHROUGH */
2510         case XPT_ATA_IO:
2511                 if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2512                         start_ccb->ataio.resid = 0;
2513                 /* FALLTHROUGH */
2514         case XPT_RESET_DEV:
2515         case XPT_ENG_EXEC:
2516         case XPT_SMP_IO:
2517         {
2518                 struct cam_devq *devq;
2519
2520                 devq = path->bus->sim->devq;
2521                 mtx_lock(&devq->send_mtx);
2522                 cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2523                 if (xpt_schedule_devq(devq, path->device) != 0)
2524                         xpt_run_devq(devq);
2525                 mtx_unlock(&devq->send_mtx);
2526                 break;
2527         }
2528         case XPT_CALC_GEOMETRY:
2529                 /* Filter out garbage */
2530                 if (start_ccb->ccg.block_size == 0
2531                  || start_ccb->ccg.volume_size == 0) {
2532                         start_ccb->ccg.cylinders = 0;
2533                         start_ccb->ccg.heads = 0;
2534                         start_ccb->ccg.secs_per_track = 0;
2535                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2536                         break;
2537                 }
2538 #if defined(PC98) || defined(__sparc64__)
2539                 /*
2540                  * In a PC-98 system, geometry translation depens on
2541                  * the "real" device geometry obtained from mode page 4.
2542                  * SCSI geometry translation is performed in the
2543                  * initialization routine of the SCSI BIOS and the result
2544                  * stored in host memory.  If the translation is available
2545                  * in host memory, use it.  If not, rely on the default
2546                  * translation the device driver performs.
2547                  * For sparc64, we may need adjust the geometry of large
2548                  * disks in order to fit the limitations of the 16-bit
2549                  * fields of the VTOC8 disk label.
2550                  */
2551                 if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2552                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2553                         break;
2554                 }
2555 #endif
2556                 goto call_sim;
2557         case XPT_ABORT:
2558         {
2559                 union ccb* abort_ccb;
2560
2561                 abort_ccb = start_ccb->cab.abort_ccb;
2562                 if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2563
2564                         if (abort_ccb->ccb_h.pinfo.index >= 0) {
2565                                 struct cam_ccbq *ccbq;
2566                                 struct cam_ed *device;
2567
2568                                 device = abort_ccb->ccb_h.path->device;
2569                                 ccbq = &device->ccbq;
2570                                 cam_ccbq_remove_ccb(ccbq, abort_ccb);
2571                                 abort_ccb->ccb_h.status =
2572                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2573                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2574                                 xpt_done(abort_ccb);
2575                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2576                                 break;
2577                         }
2578                         if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2579                          && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2580                                 /*
2581                                  * We've caught this ccb en route to
2582                                  * the SIM.  Flag it for abort and the
2583                                  * SIM will do so just before starting
2584                                  * real work on the CCB.
2585                                  */
2586                                 abort_ccb->ccb_h.status =
2587                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2588                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2589                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2590                                 break;
2591                         }
2592                 }
2593                 if (XPT_FC_IS_QUEUED(abort_ccb)
2594                  && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2595                         /*
2596                          * It's already completed but waiting
2597                          * for our SWI to get to it.
2598                          */
2599                         start_ccb->ccb_h.status = CAM_UA_ABORT;
2600                         break;
2601                 }
2602                 /*
2603                  * If we weren't able to take care of the abort request
2604                  * in the XPT, pass the request down to the SIM for processing.
2605                  */
2606         }
2607         /* FALLTHROUGH */
2608         case XPT_ACCEPT_TARGET_IO:
2609         case XPT_EN_LUN:
2610         case XPT_IMMED_NOTIFY:
2611         case XPT_NOTIFY_ACK:
2612         case XPT_RESET_BUS:
2613         case XPT_IMMEDIATE_NOTIFY:
2614         case XPT_NOTIFY_ACKNOWLEDGE:
2615         case XPT_GET_SIM_KNOB:
2616         case XPT_SET_SIM_KNOB:
2617         case XPT_GET_TRAN_SETTINGS:
2618         case XPT_SET_TRAN_SETTINGS:
2619         case XPT_PATH_INQ:
2620 call_sim:
2621                 sim = path->bus->sim;
2622                 lock = (mtx_owned(sim->mtx) == 0);
2623                 if (lock)
2624                         CAM_SIM_LOCK(sim);
2625                 (*(sim->sim_action))(sim, start_ccb);
2626                 if (lock)
2627                         CAM_SIM_UNLOCK(sim);
2628                 break;
2629         case XPT_PATH_STATS:
2630                 start_ccb->cpis.last_reset = path->bus->last_reset;
2631                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2632                 break;
2633         case XPT_GDEV_TYPE:
2634         {
2635                 struct cam_ed *dev;
2636
2637                 dev = path->device;
2638                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2639                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2640                 } else {
2641                         struct ccb_getdev *cgd;
2642
2643                         cgd = &start_ccb->cgd;
2644                         cgd->protocol = dev->protocol;
2645                         cgd->inq_data = dev->inq_data;
2646                         cgd->ident_data = dev->ident_data;
2647                         cgd->inq_flags = dev->inq_flags;
2648                         cgd->ccb_h.status = CAM_REQ_CMP;
2649                         cgd->serial_num_len = dev->serial_num_len;
2650                         if ((dev->serial_num_len > 0)
2651                          && (dev->serial_num != NULL))
2652                                 bcopy(dev->serial_num, cgd->serial_num,
2653                                       dev->serial_num_len);
2654                 }
2655                 break;
2656         }
2657         case XPT_GDEV_STATS:
2658         {
2659                 struct cam_ed *dev;
2660
2661                 dev = path->device;
2662                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2663                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2664                 } else {
2665                         struct ccb_getdevstats *cgds;
2666                         struct cam_eb *bus;
2667                         struct cam_et *tar;
2668                         struct cam_devq *devq;
2669
2670                         cgds = &start_ccb->cgds;
2671                         bus = path->bus;
2672                         tar = path->target;
2673                         devq = bus->sim->devq;
2674                         mtx_lock(&devq->send_mtx);
2675                         cgds->dev_openings = dev->ccbq.dev_openings;
2676                         cgds->dev_active = dev->ccbq.dev_active;
2677                         cgds->allocated = dev->ccbq.allocated;
2678                         cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2679                         cgds->held = cgds->allocated - cgds->dev_active -
2680                             cgds->queued;
2681                         cgds->last_reset = tar->last_reset;
2682                         cgds->maxtags = dev->maxtags;
2683                         cgds->mintags = dev->mintags;
2684                         if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2685                                 cgds->last_reset = bus->last_reset;
2686                         mtx_unlock(&devq->send_mtx);
2687                         cgds->ccb_h.status = CAM_REQ_CMP;
2688                 }
2689                 break;
2690         }
2691         case XPT_GDEVLIST:
2692         {
2693                 struct cam_periph       *nperiph;
2694                 struct periph_list      *periph_head;
2695                 struct ccb_getdevlist   *cgdl;
2696                 u_int                   i;
2697                 struct cam_ed           *device;
2698                 int                     found;
2699
2700
2701                 found = 0;
2702
2703                 /*
2704                  * Don't want anyone mucking with our data.
2705                  */
2706                 device = path->device;
2707                 periph_head = &device->periphs;
2708                 cgdl = &start_ccb->cgdl;
2709
2710                 /*
2711                  * Check and see if the list has changed since the user
2712                  * last requested a list member.  If so, tell them that the
2713                  * list has changed, and therefore they need to start over
2714                  * from the beginning.
2715                  */
2716                 if ((cgdl->index != 0) &&
2717                     (cgdl->generation != device->generation)) {
2718                         cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2719                         break;
2720                 }
2721
2722                 /*
2723                  * Traverse the list of peripherals and attempt to find
2724                  * the requested peripheral.
2725                  */
2726                 for (nperiph = SLIST_FIRST(periph_head), i = 0;
2727                      (nperiph != NULL) && (i <= cgdl->index);
2728                      nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2729                         if (i == cgdl->index) {
2730                                 strncpy(cgdl->periph_name,
2731                                         nperiph->periph_name,
2732                                         DEV_IDLEN);
2733                                 cgdl->unit_number = nperiph->unit_number;
2734                                 found = 1;
2735                         }
2736                 }
2737                 if (found == 0) {
2738                         cgdl->status = CAM_GDEVLIST_ERROR;
2739                         break;
2740                 }
2741
2742                 if (nperiph == NULL)
2743                         cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2744                 else
2745                         cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2746
2747                 cgdl->index++;
2748                 cgdl->generation = device->generation;
2749
2750                 cgdl->ccb_h.status = CAM_REQ_CMP;
2751                 break;
2752         }
2753         case XPT_DEV_MATCH:
2754         {
2755                 dev_pos_type position_type;
2756                 struct ccb_dev_match *cdm;
2757
2758                 cdm = &start_ccb->cdm;
2759
2760                 /*
2761                  * There are two ways of getting at information in the EDT.
2762                  * The first way is via the primary EDT tree.  It starts
2763                  * with a list of busses, then a list of targets on a bus,
2764                  * then devices/luns on a target, and then peripherals on a
2765                  * device/lun.  The "other" way is by the peripheral driver
2766                  * lists.  The peripheral driver lists are organized by
2767                  * peripheral driver.  (obviously)  So it makes sense to
2768                  * use the peripheral driver list if the user is looking
2769                  * for something like "da1", or all "da" devices.  If the
2770                  * user is looking for something on a particular bus/target
2771                  * or lun, it's generally better to go through the EDT tree.
2772                  */
2773
2774                 if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2775                         position_type = cdm->pos.position_type;
2776                 else {
2777                         u_int i;
2778
2779                         position_type = CAM_DEV_POS_NONE;
2780
2781                         for (i = 0; i < cdm->num_patterns; i++) {
2782                                 if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2783                                  ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2784                                         position_type = CAM_DEV_POS_EDT;
2785                                         break;
2786                                 }
2787                         }
2788
2789                         if (cdm->num_patterns == 0)
2790                                 position_type = CAM_DEV_POS_EDT;
2791                         else if (position_type == CAM_DEV_POS_NONE)
2792                                 position_type = CAM_DEV_POS_PDRV;
2793                 }
2794
2795                 switch(position_type & CAM_DEV_POS_TYPEMASK) {
2796                 case CAM_DEV_POS_EDT:
2797                         xptedtmatch(cdm);
2798                         break;
2799                 case CAM_DEV_POS_PDRV:
2800                         xptperiphlistmatch(cdm);
2801                         break;
2802                 default:
2803                         cdm->status = CAM_DEV_MATCH_ERROR;
2804                         break;
2805                 }
2806
2807                 if (cdm->status == CAM_DEV_MATCH_ERROR)
2808                         start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2809                 else
2810                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2811
2812                 break;
2813         }
2814         case XPT_SASYNC_CB:
2815         {
2816                 struct ccb_setasync *csa;
2817                 struct async_node *cur_entry;
2818                 struct async_list *async_head;
2819                 u_int32_t added;
2820
2821                 csa = &start_ccb->csa;
2822                 added = csa->event_enable;
2823                 async_head = &path->device->asyncs;
2824
2825                 /*
2826                  * If there is already an entry for us, simply
2827                  * update it.
2828                  */
2829                 cur_entry = SLIST_FIRST(async_head);
2830                 while (cur_entry != NULL) {
2831                         if ((cur_entry->callback_arg == csa->callback_arg)
2832                          && (cur_entry->callback == csa->callback))
2833                                 break;
2834                         cur_entry = SLIST_NEXT(cur_entry, links);
2835                 }
2836
2837                 if (cur_entry != NULL) {
2838                         /*
2839                          * If the request has no flags set,
2840                          * remove the entry.
2841                          */
2842                         added &= ~cur_entry->event_enable;
2843                         if (csa->event_enable == 0) {
2844                                 SLIST_REMOVE(async_head, cur_entry,
2845                                              async_node, links);
2846                                 xpt_release_device(path->device);
2847                                 free(cur_entry, M_CAMXPT);
2848                         } else {
2849                                 cur_entry->event_enable = csa->event_enable;
2850                         }
2851                         csa->event_enable = added;
2852                 } else {
2853                         cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2854                                            M_NOWAIT);
2855                         if (cur_entry == NULL) {
2856                                 csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2857                                 break;
2858                         }
2859                         cur_entry->event_enable = csa->event_enable;
2860                         cur_entry->event_lock =
2861                             mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2862                         cur_entry->callback_arg = csa->callback_arg;
2863                         cur_entry->callback = csa->callback;
2864                         SLIST_INSERT_HEAD(async_head, cur_entry, links);
2865                         xpt_acquire_device(path->device);
2866                 }
2867                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2868                 break;
2869         }
2870         case XPT_REL_SIMQ:
2871         {
2872                 struct ccb_relsim *crs;
2873                 struct cam_ed *dev;
2874
2875                 crs = &start_ccb->crs;
2876                 dev = path->device;
2877                 if (dev == NULL) {
2878
2879                         crs->ccb_h.status = CAM_DEV_NOT_THERE;
2880                         break;
2881                 }
2882
2883                 if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2884
2885                         /* Don't ever go below one opening */
2886                         if (crs->openings > 0) {
2887                                 xpt_dev_ccbq_resize(path, crs->openings);
2888                                 if (bootverbose) {
2889                                         xpt_print(path,
2890                                             "number of openings is now %d\n",
2891                                             crs->openings);
2892                                 }
2893                         }
2894                 }
2895
2896                 mtx_lock(&dev->sim->devq->send_mtx);
2897                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2898
2899                         if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2900
2901                                 /*
2902                                  * Just extend the old timeout and decrement
2903                                  * the freeze count so that a single timeout
2904                                  * is sufficient for releasing the queue.
2905                                  */
2906                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2907                                 callout_stop(&dev->callout);
2908                         } else {
2909
2910                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2911                         }
2912
2913                         callout_reset_sbt(&dev->callout,
2914                             SBT_1MS * crs->release_timeout, 0,
2915                             xpt_release_devq_timeout, dev, 0);
2916
2917                         dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2918
2919                 }
2920
2921                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2922
2923                         if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2924                                 /*
2925                                  * Decrement the freeze count so that a single
2926                                  * completion is still sufficient to unfreeze
2927                                  * the queue.
2928                                  */
2929                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2930                         } else {
2931
2932                                 dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2933                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2934                         }
2935                 }
2936
2937                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2938
2939                         if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2940                          || (dev->ccbq.dev_active == 0)) {
2941
2942                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2943                         } else {
2944
2945                                 dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2946                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2947                         }
2948                 }
2949                 mtx_unlock(&dev->sim->devq->send_mtx);
2950
2951                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2952                         xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2953                 start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2954                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2955                 break;
2956         }
2957         case XPT_DEBUG: {
2958                 struct cam_path *oldpath;
2959
2960                 /* Check that all request bits are supported. */
2961                 if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2962                         start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2963                         break;
2964                 }
2965
2966                 cam_dflags = CAM_DEBUG_NONE;
2967                 if (cam_dpath != NULL) {
2968                         oldpath = cam_dpath;
2969                         cam_dpath = NULL;
2970                         xpt_free_path(oldpath);
2971                 }
2972                 if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2973                         if (xpt_create_path(&cam_dpath, NULL,
2974                                             start_ccb->ccb_h.path_id,
2975                                             start_ccb->ccb_h.target_id,
2976                                             start_ccb->ccb_h.target_lun) !=
2977                                             CAM_REQ_CMP) {
2978                                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2979                         } else {
2980                                 cam_dflags = start_ccb->cdbg.flags;
2981                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2982                                 xpt_print(cam_dpath, "debugging flags now %x\n",
2983                                     cam_dflags);
2984                         }
2985                 } else
2986                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2987                 break;
2988         }
2989         case XPT_NOOP:
2990                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2991                         xpt_freeze_devq(path, 1);
2992                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2993                 break;
2994         default:
2995         case XPT_SDEV_TYPE:
2996         case XPT_TERM_IO:
2997         case XPT_ENG_INQ:
2998                 /* XXX Implement */
2999                 printf("%s: CCB type %#x not supported\n", __func__,
3000                        start_ccb->ccb_h.func_code);
3001                 start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3002                 if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3003                         xpt_done(start_ccb);
3004                 }
3005                 break;
3006         }
3007 }
3008
3009 void
3010 xpt_polled_action(union ccb *start_ccb)
3011 {
3012         u_int32_t timeout;
3013         struct    cam_sim *sim;
3014         struct    cam_devq *devq;
3015         struct    cam_ed *dev;
3016
3017         timeout = start_ccb->ccb_h.timeout * 10;
3018         sim = start_ccb->ccb_h.path->bus->sim;
3019         devq = sim->devq;
3020         dev = start_ccb->ccb_h.path->device;
3021
3022         mtx_unlock(&dev->device_mtx);
3023
3024         /*
3025          * Steal an opening so that no other queued requests
3026          * can get it before us while we simulate interrupts.
3027          */
3028         mtx_lock(&devq->send_mtx);
3029         dev->ccbq.dev_openings--;
3030         while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3031             (--timeout > 0)) {
3032                 mtx_unlock(&devq->send_mtx);
3033                 DELAY(100);
3034                 CAM_SIM_LOCK(sim);
3035                 (*(sim->sim_poll))(sim);
3036                 CAM_SIM_UNLOCK(sim);
3037                 camisr_runqueue();
3038                 mtx_lock(&devq->send_mtx);
3039         }
3040         dev->ccbq.dev_openings++;
3041         mtx_unlock(&devq->send_mtx);
3042
3043         if (timeout != 0) {
3044                 xpt_action(start_ccb);
3045                 while(--timeout > 0) {
3046                         CAM_SIM_LOCK(sim);
3047                         (*(sim->sim_poll))(sim);
3048                         CAM_SIM_UNLOCK(sim);
3049                         camisr_runqueue();
3050                         if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3051                             != CAM_REQ_INPROG)
3052                                 break;
3053                         DELAY(100);
3054                 }
3055                 if (timeout == 0) {
3056                         /*
3057                          * XXX Is it worth adding a sim_timeout entry
3058                          * point so we can attempt recovery?  If
3059                          * this is only used for dumps, I don't think
3060                          * it is.
3061                          */
3062                         start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3063                 }
3064         } else {
3065                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3066         }
3067
3068         mtx_lock(&dev->device_mtx);
3069 }
3070
3071 /*
3072  * Schedule a peripheral driver to receive a ccb when its
3073  * target device has space for more transactions.
3074  */
3075 void
3076 xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3077 {
3078
3079         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3080         cam_periph_assert(periph, MA_OWNED);
3081         if (new_priority < periph->scheduled_priority) {
3082                 periph->scheduled_priority = new_priority;
3083                 xpt_run_allocq(periph, 0);
3084         }
3085 }
3086
3087
3088 /*
3089  * Schedule a device to run on a given queue.
3090  * If the device was inserted as a new entry on the queue,
3091  * return 1 meaning the device queue should be run. If we
3092  * were already queued, implying someone else has already
3093  * started the queue, return 0 so the caller doesn't attempt
3094  * to run the queue.
3095  */
3096 static int
3097 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3098                  u_int32_t new_priority)
3099 {
3100         int retval;
3101         u_int32_t old_priority;
3102
3103         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3104
3105         old_priority = pinfo->priority;
3106
3107         /*
3108          * Are we already queued?
3109          */
3110         if (pinfo->index != CAM_UNQUEUED_INDEX) {
3111                 /* Simply reorder based on new priority */
3112                 if (new_priority < old_priority) {
3113                         camq_change_priority(queue, pinfo->index,
3114                                              new_priority);
3115                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3116                                         ("changed priority to %d\n",
3117                                          new_priority));
3118                         retval = 1;
3119                 } else
3120                         retval = 0;
3121         } else {
3122                 /* New entry on the queue */
3123                 if (new_priority < old_priority)
3124                         pinfo->priority = new_priority;
3125
3126                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3127                                 ("Inserting onto queue\n"));
3128                 pinfo->generation = ++queue->generation;
3129                 camq_insert(queue, pinfo);
3130                 retval = 1;
3131         }
3132         return (retval);
3133 }
3134
3135 static void
3136 xpt_run_allocq_task(void *context, int pending)
3137 {
3138         struct cam_periph *periph = context;
3139
3140         cam_periph_lock(periph);
3141         periph->flags &= ~CAM_PERIPH_RUN_TASK;
3142         xpt_run_allocq(periph, 1);
3143         cam_periph_unlock(periph);
3144         cam_periph_release(periph);
3145 }
3146
3147 static void
3148 xpt_run_allocq(struct cam_periph *periph, int sleep)
3149 {
3150         struct cam_ed   *device;
3151         union ccb       *ccb;
3152         uint32_t         prio;
3153
3154         cam_periph_assert(periph, MA_OWNED);
3155         if (periph->periph_allocating)
3156                 return;
3157         periph->periph_allocating = 1;
3158         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3159         device = periph->path->device;
3160         ccb = NULL;
3161 restart:
3162         while ((prio = min(periph->scheduled_priority,
3163             periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3164             (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3165              device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3166
3167                 if (ccb == NULL &&
3168                     (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3169                         if (sleep) {
3170                                 ccb = xpt_get_ccb(periph);
3171                                 goto restart;
3172                         }
3173                         if (periph->flags & CAM_PERIPH_RUN_TASK)
3174                                 break;
3175                         cam_periph_doacquire(periph);
3176                         periph->flags |= CAM_PERIPH_RUN_TASK;
3177                         taskqueue_enqueue(xsoftc.xpt_taskq,
3178                             &periph->periph_run_task);
3179                         break;
3180                 }
3181                 xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3182                 if (prio == periph->immediate_priority) {
3183                         periph->immediate_priority = CAM_PRIORITY_NONE;
3184                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3185                                         ("waking cam_periph_getccb()\n"));
3186                         SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3187                                           periph_links.sle);
3188                         wakeup(&periph->ccb_list);
3189                 } else {
3190                         periph->scheduled_priority = CAM_PRIORITY_NONE;
3191                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3192                                         ("calling periph_start()\n"));
3193                         periph->periph_start(periph, ccb);
3194                 }
3195                 ccb = NULL;
3196         }
3197         if (ccb != NULL)
3198                 xpt_release_ccb(ccb);
3199         periph->periph_allocating = 0;
3200 }
3201
3202 static void
3203 xpt_run_devq(struct cam_devq *devq)
3204 {
3205         char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3206         int lock;
3207
3208         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3209
3210         devq->send_queue.qfrozen_cnt++;
3211         while ((devq->send_queue.entries > 0)
3212             && (devq->send_openings > 0)
3213             && (devq->send_queue.qfrozen_cnt <= 1)) {
3214                 struct  cam_ed *device;
3215                 union ccb *work_ccb;
3216                 struct  cam_sim *sim;
3217
3218                 device = (struct cam_ed *)camq_remove(&devq->send_queue,
3219                                                            CAMQ_HEAD);
3220                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3221                                 ("running device %p\n", device));
3222
3223                 work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3224                 if (work_ccb == NULL) {
3225                         printf("device on run queue with no ccbs???\n");
3226                         continue;
3227                 }
3228
3229                 if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3230
3231                         mtx_lock(&xsoftc.xpt_highpower_lock);
3232                         if (xsoftc.num_highpower <= 0) {
3233                                 /*
3234                                  * We got a high power command, but we
3235                                  * don't have any available slots.  Freeze
3236                                  * the device queue until we have a slot
3237                                  * available.
3238                                  */
3239                                 xpt_freeze_devq_device(device, 1);
3240                                 STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3241                                                    highpowerq_entry);
3242
3243                                 mtx_unlock(&xsoftc.xpt_highpower_lock);
3244                                 continue;
3245                         } else {
3246                                 /*
3247                                  * Consume a high power slot while
3248                                  * this ccb runs.
3249                                  */
3250                                 xsoftc.num_highpower--;
3251                         }
3252                         mtx_unlock(&xsoftc.xpt_highpower_lock);
3253                 }
3254                 cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3255                 cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3256                 devq->send_openings--;
3257                 devq->send_active++;
3258                 xpt_schedule_devq(devq, device);
3259                 mtx_unlock(&devq->send_mtx);
3260
3261                 if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3262                         /*
3263                          * The client wants to freeze the queue
3264                          * after this CCB is sent.
3265                          */
3266                         xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3267                 }
3268
3269                 /* In Target mode, the peripheral driver knows best... */
3270                 if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3271                         if ((device->inq_flags & SID_CmdQue) != 0
3272                          && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3273                                 work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3274                         else
3275                                 /*
3276                                  * Clear this in case of a retried CCB that
3277                                  * failed due to a rejected tag.
3278                                  */
3279                                 work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3280                 }
3281
3282                 switch (work_ccb->ccb_h.func_code) {
3283                 case XPT_SCSI_IO:
3284                         CAM_DEBUG(work_ccb->ccb_h.path,
3285                             CAM_DEBUG_CDB,("%s. CDB: %s\n",
3286                              scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3287                                           &device->inq_data),
3288                              scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3289                                              cdb_str, sizeof(cdb_str))));
3290                         break;
3291                 case XPT_ATA_IO:
3292                         CAM_DEBUG(work_ccb->ccb_h.path,
3293                             CAM_DEBUG_CDB,("%s. ACB: %s\n",
3294                              ata_op_string(&work_ccb->ataio.cmd),
3295                              ata_cmd_string(&work_ccb->ataio.cmd,
3296                                             cdb_str, sizeof(cdb_str))));
3297                         break;
3298                 default:
3299                         break;
3300                 }
3301
3302                 /*
3303                  * Device queues can be shared among multiple SIM instances
3304                  * that reside on different busses.  Use the SIM from the
3305                  * queued device, rather than the one from the calling bus.
3306                  */
3307                 sim = device->sim;
3308                 lock = (mtx_owned(sim->mtx) == 0);
3309                 if (lock)
3310                         CAM_SIM_LOCK(sim);
3311                 (*(sim->sim_action))(sim, work_ccb);
3312                 if (lock)
3313                         CAM_SIM_UNLOCK(sim);
3314                 mtx_lock(&devq->send_mtx);
3315         }
3316         devq->send_queue.qfrozen_cnt--;
3317 }
3318
3319 /*
3320  * This function merges stuff from the slave ccb into the master ccb, while
3321  * keeping important fields in the master ccb constant.
3322  */
3323 void
3324 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3325 {
3326
3327         /*
3328          * Pull fields that are valid for peripheral drivers to set
3329          * into the master CCB along with the CCB "payload".
3330          */
3331         master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3332         master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3333         master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3334         master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3335         bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3336               sizeof(union ccb) - sizeof(struct ccb_hdr));
3337 }
3338
3339 void
3340 xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3341                     u_int32_t priority, u_int32_t flags)
3342 {
3343
3344         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3345         ccb_h->pinfo.priority = priority;
3346         ccb_h->path = path;
3347         ccb_h->path_id = path->bus->path_id;
3348         if (path->target)
3349                 ccb_h->target_id = path->target->target_id;
3350         else
3351                 ccb_h->target_id = CAM_TARGET_WILDCARD;
3352         if (path->device) {
3353                 ccb_h->target_lun = path->device->lun_id;
3354                 ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3355         } else {
3356                 ccb_h->target_lun = CAM_TARGET_WILDCARD;
3357         }
3358         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3359         ccb_h->flags = flags;
3360         ccb_h->xflags = 0;
3361 }
3362
3363 void
3364 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3365 {
3366         xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3367 }
3368
3369 /* Path manipulation functions */
3370 cam_status
3371 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3372                 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3373 {
3374         struct     cam_path *path;
3375         cam_status status;
3376
3377         path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3378
3379         if (path == NULL) {
3380                 status = CAM_RESRC_UNAVAIL;
3381                 return(status);
3382         }
3383         status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3384         if (status != CAM_REQ_CMP) {
3385                 free(path, M_CAMPATH);
3386                 path = NULL;
3387         }
3388         *new_path_ptr = path;
3389         return (status);
3390 }
3391
3392 cam_status
3393 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3394                          struct cam_periph *periph, path_id_t path_id,
3395                          target_id_t target_id, lun_id_t lun_id)
3396 {
3397
3398         return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3399             lun_id));
3400 }
3401
3402 cam_status
3403 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3404                  path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3405 {
3406         struct       cam_eb *bus;
3407         struct       cam_et *target;
3408         struct       cam_ed *device;
3409         cam_status   status;
3410
3411         status = CAM_REQ_CMP;   /* Completed without error */
3412         target = NULL;          /* Wildcarded */
3413         device = NULL;          /* Wildcarded */
3414
3415         /*
3416          * We will potentially modify the EDT, so block interrupts
3417          * that may attempt to create cam paths.
3418          */
3419         bus = xpt_find_bus(path_id);
3420         if (bus == NULL) {
3421                 status = CAM_PATH_INVALID;
3422         } else {
3423                 xpt_lock_buses();
3424                 mtx_lock(&bus->eb_mtx);
3425                 target = xpt_find_target(bus, target_id);
3426                 if (target == NULL) {
3427                         /* Create one */
3428                         struct cam_et *new_target;
3429
3430                         new_target = xpt_alloc_target(bus, target_id);
3431                         if (new_target == NULL) {
3432                                 status = CAM_RESRC_UNAVAIL;
3433                         } else {
3434                                 target = new_target;
3435                         }
3436                 }
3437                 xpt_unlock_buses();
3438                 if (target != NULL) {
3439                         device = xpt_find_device(target, lun_id);
3440                         if (device == NULL) {
3441                                 /* Create one */
3442                                 struct cam_ed *new_device;
3443
3444                                 new_device =
3445                                     (*(bus->xport->alloc_device))(bus,
3446                                                                       target,
3447                                                                       lun_id);
3448                                 if (new_device == NULL) {
3449                                         status = CAM_RESRC_UNAVAIL;
3450                                 } else {
3451                                         device = new_device;
3452                                 }
3453                         }
3454                 }
3455                 mtx_unlock(&bus->eb_mtx);
3456         }
3457
3458         /*
3459          * Only touch the user's data if we are successful.
3460          */
3461         if (status == CAM_REQ_CMP) {
3462                 new_path->periph = perph;
3463                 new_path->bus = bus;
3464                 new_path->target = target;
3465                 new_path->device = device;
3466                 CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3467         } else {
3468                 if (device != NULL)
3469                         xpt_release_device(device);
3470                 if (target != NULL)
3471                         xpt_release_target(target);
3472                 if (bus != NULL)
3473                         xpt_release_bus(bus);
3474         }
3475         return (status);
3476 }
3477
3478 cam_status
3479 xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3480 {
3481         struct     cam_path *new_path;
3482
3483         new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3484         if (new_path == NULL)
3485                 return(CAM_RESRC_UNAVAIL);
3486         xpt_copy_path(new_path, path);
3487         *new_path_ptr = new_path;
3488         return (CAM_REQ_CMP);
3489 }
3490
3491 void
3492 xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3493 {
3494
3495         *new_path = *path;
3496         if (path->bus != NULL)
3497                 xpt_acquire_bus(path->bus);
3498         if (path->target != NULL)
3499                 xpt_acquire_target(path->target);
3500         if (path->device != NULL)
3501                 xpt_acquire_device(path->device);
3502 }
3503
3504 void
3505 xpt_release_path(struct cam_path *path)
3506 {
3507         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3508         if (path->device != NULL) {
3509                 xpt_release_device(path->device);
3510                 path->device = NULL;
3511         }
3512         if (path->target != NULL) {
3513                 xpt_release_target(path->target);
3514                 path->target = NULL;
3515         }
3516         if (path->bus != NULL) {
3517                 xpt_release_bus(path->bus);
3518                 path->bus = NULL;
3519         }
3520 }
3521
3522 void
3523 xpt_free_path(struct cam_path *path)
3524 {
3525
3526         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3527         xpt_release_path(path);
3528         free(path, M_CAMPATH);
3529 }
3530
3531 void
3532 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3533     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3534 {
3535
3536         xpt_lock_buses();
3537         if (bus_ref) {
3538                 if (path->bus)
3539                         *bus_ref = path->bus->refcount;
3540                 else
3541                         *bus_ref = 0;
3542         }
3543         if (periph_ref) {
3544                 if (path->periph)
3545                         *periph_ref = path->periph->refcount;
3546                 else
3547                         *periph_ref = 0;
3548         }
3549         xpt_unlock_buses();
3550         if (target_ref) {
3551                 if (path->target)
3552                         *target_ref = path->target->refcount;
3553                 else
3554                         *target_ref = 0;
3555         }
3556         if (device_ref) {
3557                 if (path->device)
3558                         *device_ref = path->device->refcount;
3559                 else
3560                         *device_ref = 0;
3561         }
3562 }
3563
3564 /*
3565  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3566  * in path1, 2 for match with wildcards in path2.
3567  */
3568 int
3569 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3570 {
3571         int retval = 0;
3572
3573         if (path1->bus != path2->bus) {
3574                 if (path1->bus->path_id == CAM_BUS_WILDCARD)
3575                         retval = 1;
3576                 else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3577                         retval = 2;
3578                 else
3579                         return (-1);
3580         }
3581         if (path1->target != path2->target) {
3582                 if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3583                         if (retval == 0)
3584                                 retval = 1;
3585                 } else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3586                         retval = 2;
3587                 else
3588                         return (-1);
3589         }
3590         if (path1->device != path2->device) {
3591                 if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3592                         if (retval == 0)
3593                                 retval = 1;
3594                 } else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3595                         retval = 2;
3596                 else
3597                         return (-1);
3598         }
3599         return (retval);
3600 }
3601
3602 int
3603 xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3604 {
3605         int retval = 0;
3606
3607         if (path->bus != dev->target->bus) {
3608                 if (path->bus->path_id == CAM_BUS_WILDCARD)
3609                         retval = 1;
3610                 else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3611                         retval = 2;
3612                 else
3613                         return (-1);
3614         }
3615         if (path->target != dev->target) {
3616                 if (path->target->target_id == CAM_TARGET_WILDCARD) {
3617                         if (retval == 0)
3618                                 retval = 1;
3619                 } else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3620                         retval = 2;
3621                 else
3622                         return (-1);
3623         }
3624         if (path->device != dev) {
3625                 if (path->device->lun_id == CAM_LUN_WILDCARD) {
3626                         if (retval == 0)
3627                                 retval = 1;
3628                 } else if (dev->lun_id == CAM_LUN_WILDCARD)
3629                         retval = 2;
3630                 else
3631                         return (-1);
3632         }
3633         return (retval);
3634 }
3635
3636 void
3637 xpt_print_path(struct cam_path *path)
3638 {
3639
3640         if (path == NULL)
3641                 printf("(nopath): ");
3642         else {
3643                 if (path->periph != NULL)
3644                         printf("(%s%d:", path->periph->periph_name,
3645                                path->periph->unit_number);
3646                 else
3647                         printf("(noperiph:");
3648
3649                 if (path->bus != NULL)
3650                         printf("%s%d:%d:", path->bus->sim->sim_name,
3651                                path->bus->sim->unit_number,
3652                                path->bus->sim->bus_id);
3653                 else
3654                         printf("nobus:");
3655
3656                 if (path->target != NULL)
3657                         printf("%d:", path->target->target_id);
3658                 else
3659                         printf("X:");
3660
3661                 if (path->device != NULL)
3662                         printf("%jx): ", (uintmax_t)path->device->lun_id);
3663                 else
3664                         printf("X): ");
3665         }
3666 }
3667
3668 void
3669 xpt_print_device(struct cam_ed *device)
3670 {
3671
3672         if (device == NULL)
3673                 printf("(nopath): ");
3674         else {
3675                 printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3676                        device->sim->unit_number,
3677                        device->sim->bus_id,
3678                        device->target->target_id,
3679                        (uintmax_t)device->lun_id);
3680         }
3681 }
3682
3683 void
3684 xpt_print(struct cam_path *path, const char *fmt, ...)
3685 {
3686         va_list ap;
3687         xpt_print_path(path);
3688         va_start(ap, fmt);
3689         vprintf(fmt, ap);
3690         va_end(ap);
3691 }
3692
3693 int
3694 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3695 {
3696         struct sbuf sb;
3697
3698         sbuf_new(&sb, str, str_len, 0);
3699
3700         if (path == NULL)
3701                 sbuf_printf(&sb, "(nopath): ");
3702         else {
3703                 if (path->periph != NULL)
3704                         sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3705                                     path->periph->unit_number);
3706                 else
3707                         sbuf_printf(&sb, "(noperiph:");
3708
3709                 if (path->bus != NULL)
3710                         sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3711                                     path->bus->sim->unit_number,
3712                                     path->bus->sim->bus_id);
3713                 else
3714                         sbuf_printf(&sb, "nobus:");
3715
3716                 if (path->target != NULL)
3717                         sbuf_printf(&sb, "%d:", path->target->target_id);
3718                 else
3719                         sbuf_printf(&sb, "X:");
3720
3721                 if (path->device != NULL)
3722                         sbuf_printf(&sb, "%jx): ",
3723                             (uintmax_t)path->device->lun_id);
3724                 else
3725                         sbuf_printf(&sb, "X): ");
3726         }
3727         sbuf_finish(&sb);
3728
3729         return(sbuf_len(&sb));
3730 }
3731
3732 path_id_t
3733 xpt_path_path_id(struct cam_path *path)
3734 {
3735         return(path->bus->path_id);
3736 }
3737
3738 target_id_t
3739 xpt_path_target_id(struct cam_path *path)
3740 {
3741         if (path->target != NULL)
3742                 return (path->target->target_id);
3743         else
3744                 return (CAM_TARGET_WILDCARD);
3745 }
3746
3747 lun_id_t
3748 xpt_path_lun_id(struct cam_path *path)
3749 {
3750         if (path->device != NULL)
3751                 return (path->device->lun_id);
3752         else
3753                 return (CAM_LUN_WILDCARD);
3754 }
3755
3756 struct cam_sim *
3757 xpt_path_sim(struct cam_path *path)
3758 {
3759
3760         return (path->bus->sim);
3761 }
3762
3763 struct cam_periph*
3764 xpt_path_periph(struct cam_path *path)
3765 {
3766
3767         return (path->periph);
3768 }
3769
3770 int
3771 xpt_path_legacy_ata_id(struct cam_path *path)
3772 {
3773         struct cam_eb *bus;
3774         int bus_id;
3775
3776         if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3777             strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3778             strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3779             strcmp(path->bus->sim->sim_name, "siisch") != 0)
3780                 return (-1);
3781
3782         if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3783             path->bus->sim->unit_number < 2) {
3784                 bus_id = path->bus->sim->unit_number;
3785         } else {
3786                 bus_id = 2;
3787                 xpt_lock_buses();
3788                 TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3789                         if (bus == path->bus)
3790                                 break;
3791                         if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3792                              bus->sim->unit_number >= 2) ||
3793                             strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3794                             strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3795                             strcmp(bus->sim->sim_name, "siisch") == 0)
3796                                 bus_id++;
3797                 }
3798                 xpt_unlock_buses();
3799         }
3800         if (path->target != NULL) {
3801                 if (path->target->target_id < 2)
3802                         return (bus_id * 2 + path->target->target_id);
3803                 else
3804                         return (-1);
3805         } else
3806                 return (bus_id * 2);
3807 }
3808
3809 /*
3810  * Release a CAM control block for the caller.  Remit the cost of the structure
3811  * to the device referenced by the path.  If the this device had no 'credits'
3812  * and peripheral drivers have registered async callbacks for this notification
3813  * call them now.
3814  */
3815 void
3816 xpt_release_ccb(union ccb *free_ccb)
3817 {
3818         struct   cam_ed *device;
3819         struct   cam_periph *periph;
3820
3821         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3822         xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3823         device = free_ccb->ccb_h.path->device;
3824         periph = free_ccb->ccb_h.path->periph;
3825
3826         xpt_free_ccb(free_ccb);
3827         periph->periph_allocated--;
3828         cam_ccbq_release_opening(&device->ccbq);
3829         xpt_run_allocq(periph, 0);
3830 }
3831
3832 /* Functions accessed by SIM drivers */
3833
3834 static struct xpt_xport xport_default = {
3835         .alloc_device = xpt_alloc_device_default,
3836         .action = xpt_action_default,
3837         .async = xpt_dev_async_default,
3838 };
3839
3840 /*
3841  * A sim structure, listing the SIM entry points and instance
3842  * identification info is passed to xpt_bus_register to hook the SIM
3843  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3844  * for this new bus and places it in the array of busses and assigns
3845  * it a path_id.  The path_id may be influenced by "hard wiring"
3846  * information specified by the user.  Once interrupt services are
3847  * available, the bus will be probed.
3848  */
3849 int32_t
3850 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3851 {
3852         struct cam_eb *new_bus;
3853         struct cam_eb *old_bus;
3854         struct ccb_pathinq cpi;
3855         struct cam_path *path;
3856         cam_status status;
3857
3858         mtx_assert(sim->mtx, MA_OWNED);
3859
3860         sim->bus_id = bus;
3861         new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3862                                           M_CAMXPT, M_NOWAIT|M_ZERO);
3863         if (new_bus == NULL) {
3864                 /* Couldn't satisfy request */
3865                 return (CAM_RESRC_UNAVAIL);
3866         }
3867
3868         mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3869         TAILQ_INIT(&new_bus->et_entries);
3870         cam_sim_hold(sim);
3871         new_bus->sim = sim;
3872         timevalclear(&new_bus->last_reset);
3873         new_bus->flags = 0;
3874         new_bus->refcount = 1;  /* Held until a bus_deregister event */
3875         new_bus->generation = 0;
3876
3877         xpt_lock_buses();
3878         sim->path_id = new_bus->path_id =
3879             xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3880         old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3881         while (old_bus != NULL
3882             && old_bus->path_id < new_bus->path_id)
3883                 old_bus = TAILQ_NEXT(old_bus, links);
3884         if (old_bus != NULL)
3885                 TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3886         else
3887                 TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3888         xsoftc.bus_generation++;
3889         xpt_unlock_buses();
3890
3891         /*
3892          * Set a default transport so that a PATH_INQ can be issued to
3893          * the SIM.  This will then allow for probing and attaching of
3894          * a more appropriate transport.
3895          */
3896         new_bus->xport = &xport_default;
3897
3898         status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3899                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3900         if (status != CAM_REQ_CMP) {
3901                 xpt_release_bus(new_bus);
3902                 free(path, M_CAMXPT);
3903                 return (CAM_RESRC_UNAVAIL);
3904         }
3905
3906         xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3907         cpi.ccb_h.func_code = XPT_PATH_INQ;
3908         xpt_action((union ccb *)&cpi);
3909
3910         if (cpi.ccb_h.status == CAM_REQ_CMP) {
3911                 switch (cpi.transport) {
3912                 case XPORT_SPI:
3913                 case XPORT_SAS:
3914                 case XPORT_FC:
3915                 case XPORT_USB:
3916                 case XPORT_ISCSI:
3917                 case XPORT_SRP:
3918                 case XPORT_PPB:
3919                         new_bus->xport = scsi_get_xport();
3920                         break;
3921                 case XPORT_ATA:
3922                 case XPORT_SATA:
3923                         new_bus->xport = ata_get_xport();
3924                         break;
3925                 default:
3926                         new_bus->xport = &xport_default;
3927                         break;
3928                 }
3929         }
3930
3931         /* Notify interested parties */
3932         if (sim->path_id != CAM_XPT_PATH_ID) {
3933
3934                 xpt_async(AC_PATH_REGISTERED, path, &cpi);
3935                 if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3936                         union   ccb *scan_ccb;
3937
3938                         /* Initiate bus rescan. */
3939                         scan_ccb = xpt_alloc_ccb_nowait();
3940                         if (scan_ccb != NULL) {
3941                                 scan_ccb->ccb_h.path = path;
3942                                 scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3943                                 scan_ccb->crcn.flags = 0;
3944                                 xpt_rescan(scan_ccb);
3945                         } else {
3946                                 xpt_print(path,
3947                                           "Can't allocate CCB to scan bus\n");
3948                                 xpt_free_path(path);
3949                         }
3950                 } else
3951                         xpt_free_path(path);
3952         } else
3953                 xpt_free_path(path);
3954         return (CAM_SUCCESS);
3955 }
3956
3957 int32_t
3958 xpt_bus_deregister(path_id_t pathid)
3959 {
3960         struct cam_path bus_path;
3961         cam_status status;
3962
3963         status = xpt_compile_path(&bus_path, NULL, pathid,
3964                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3965         if (status != CAM_REQ_CMP)
3966                 return (status);
3967
3968         xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3969         xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3970
3971         /* Release the reference count held while registered. */
3972         xpt_release_bus(bus_path.bus);
3973         xpt_release_path(&bus_path);
3974
3975         return (CAM_REQ_CMP);
3976 }
3977
3978 static path_id_t
3979 xptnextfreepathid(void)
3980 {
3981         struct cam_eb *bus;
3982         path_id_t pathid;
3983         const char *strval;
3984
3985         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3986         pathid = 0;
3987         bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3988 retry:
3989         /* Find an unoccupied pathid */
3990         while (bus != NULL && bus->path_id <= pathid) {
3991                 if (bus->path_id == pathid)
3992                         pathid++;
3993                 bus = TAILQ_NEXT(bus, links);
3994         }
3995
3996         /*
3997          * Ensure that this pathid is not reserved for
3998          * a bus that may be registered in the future.
3999          */
4000         if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4001                 ++pathid;
4002                 /* Start the search over */
4003                 goto retry;
4004         }
4005         return (pathid);
4006 }
4007
4008 static path_id_t
4009 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4010 {
4011         path_id_t pathid;
4012         int i, dunit, val;
4013         char buf[32];
4014         const char *dname;
4015
4016         pathid = CAM_XPT_PATH_ID;
4017         snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4018         if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4019                 return (pathid);
4020         i = 0;
4021         while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4022                 if (strcmp(dname, "scbus")) {
4023                         /* Avoid a bit of foot shooting. */
4024                         continue;
4025                 }
4026                 if (dunit < 0)          /* unwired?! */
4027                         continue;
4028                 if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4029                         if (sim_bus == val) {
4030                                 pathid = dunit;
4031                                 break;
4032                         }
4033                 } else if (sim_bus == 0) {
4034                         /* Unspecified matches bus 0 */
4035                         pathid = dunit;
4036                         break;
4037                 } else {
4038                         printf("Ambiguous scbus configuration for %s%d "
4039                                "bus %d, cannot wire down.  The kernel "
4040                                "config entry for scbus%d should "
4041                                "specify a controller bus.\n"
4042                                "Scbus will be assigned dynamically.\n",
4043                                sim_name, sim_unit, sim_bus, dunit);
4044                         break;
4045                 }
4046         }
4047
4048         if (pathid == CAM_XPT_PATH_ID)
4049                 pathid = xptnextfreepathid();
4050         return (pathid);
4051 }
4052
4053 static const char *
4054 xpt_async_string(u_int32_t async_code)
4055 {
4056
4057         switch (async_code) {
4058         case AC_BUS_RESET: return ("AC_BUS_RESET");
4059         case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4060         case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4061         case AC_SENT_BDR: return ("AC_SENT_BDR");
4062         case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4063         case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4064         case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4065         case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4066         case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4067         case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4068         case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4069         case AC_CONTRACT: return ("AC_CONTRACT");
4070         case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4071         case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4072         }
4073         return ("AC_UNKNOWN");
4074 }
4075
4076 static int
4077 xpt_async_size(u_int32_t async_code)
4078 {
4079
4080         switch (async_code) {
4081         case AC_BUS_RESET: return (0);
4082         case AC_UNSOL_RESEL: return (0);
4083         case AC_SCSI_AEN: return (0);
4084         case AC_SENT_BDR: return (0);
4085         case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4086         case AC_PATH_DEREGISTERED: return (0);
4087         case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4088         case AC_LOST_DEVICE: return (0);
4089         case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4090         case AC_INQ_CHANGED: return (0);
4091         case AC_GETDEV_CHANGED: return (0);
4092         case AC_CONTRACT: return (sizeof(struct ac_contract));
4093         case AC_ADVINFO_CHANGED: return (-1);
4094         case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4095         }
4096         return (0);
4097 }
4098
4099 static int
4100 xpt_async_process_dev(struct cam_ed *device, void *arg)
4101 {
4102         union ccb *ccb = arg;
4103         struct cam_path *path = ccb->ccb_h.path;
4104         void *async_arg = ccb->casync.async_arg_ptr;
4105         u_int32_t async_code = ccb->casync.async_code;
4106         int relock;
4107
4108         if (path->device != device
4109          && path->device->lun_id != CAM_LUN_WILDCARD
4110          && device->lun_id != CAM_LUN_WILDCARD)
4111                 return (1);
4112
4113         /*
4114          * The async callback could free the device.
4115          * If it is a broadcast async, it doesn't hold
4116          * device reference, so take our own reference.
4117          */
4118         xpt_acquire_device(device);
4119
4120         /*
4121          * If async for specific device is to be delivered to
4122          * the wildcard client, take the specific device lock.
4123          * XXX: We may need a way for client to specify it.
4124          */
4125         if ((device->lun_id == CAM_LUN_WILDCARD &&
4126              path->device->lun_id != CAM_LUN_WILDCARD) ||
4127             (device->target->target_id == CAM_TARGET_WILDCARD &&
4128              path->target->target_id != CAM_TARGET_WILDCARD) ||
4129             (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4130              path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4131                 mtx_unlock(&device->device_mtx);
4132                 xpt_path_lock(path);
4133                 relock = 1;
4134         } else
4135                 relock = 0;
4136
4137         (*(device->target->bus->xport->async))(async_code,
4138             device->target->bus, device->target, device, async_arg);
4139         xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4140
4141         if (relock) {
4142                 xpt_path_unlock(path);
4143                 mtx_lock(&device->device_mtx);
4144         }
4145         xpt_release_device(device);
4146         return (1);
4147 }
4148
4149 static int
4150 xpt_async_process_tgt(struct cam_et *target, void *arg)
4151 {
4152         union ccb *ccb = arg;
4153         struct cam_path *path = ccb->ccb_h.path;
4154
4155         if (path->target != target
4156          && path->target->target_id != CAM_TARGET_WILDCARD
4157          && target->target_id != CAM_TARGET_WILDCARD)
4158                 return (1);
4159
4160         if (ccb->casync.async_code == AC_SENT_BDR) {
4161                 /* Update our notion of when the last reset occurred */
4162                 microtime(&target->last_reset);
4163         }
4164
4165         return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4166 }
4167
4168 static void
4169 xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4170 {
4171         struct cam_eb *bus;
4172         struct cam_path *path;
4173         void *async_arg;
4174         u_int32_t async_code;
4175
4176         path = ccb->ccb_h.path;
4177         async_code = ccb->casync.async_code;
4178         async_arg = ccb->casync.async_arg_ptr;
4179         CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4180             ("xpt_async(%s)\n", xpt_async_string(async_code)));
4181         bus = path->bus;
4182
4183         if (async_code == AC_BUS_RESET) {
4184                 /* Update our notion of when the last reset occurred */
4185                 microtime(&bus->last_reset);
4186         }
4187
4188         xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4189
4190         /*
4191          * If this wasn't a fully wildcarded async, tell all
4192          * clients that want all async events.
4193          */
4194         if (bus != xpt_periph->path->bus) {
4195                 xpt_path_lock(xpt_periph->path);
4196                 xpt_async_process_dev(xpt_periph->path->device, ccb);
4197                 xpt_path_unlock(xpt_periph->path);
4198         }
4199
4200         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4201                 xpt_release_devq(path, 1, TRUE);
4202         else
4203                 xpt_release_simq(path->bus->sim, TRUE);
4204         if (ccb->casync.async_arg_size > 0)
4205                 free(async_arg, M_CAMXPT);
4206         xpt_free_path(path);
4207         xpt_free_ccb(ccb);
4208 }
4209
4210 static void
4211 xpt_async_bcast(struct async_list *async_head,
4212                 u_int32_t async_code,
4213                 struct cam_path *path, void *async_arg)
4214 {
4215         struct async_node *cur_entry;
4216         int lock;
4217
4218         cur_entry = SLIST_FIRST(async_head);
4219         while (cur_entry != NULL) {
4220                 struct async_node *next_entry;
4221                 /*
4222                  * Grab the next list entry before we call the current
4223                  * entry's callback.  This is because the callback function
4224                  * can delete its async callback entry.
4225                  */
4226                 next_entry = SLIST_NEXT(cur_entry, links);
4227                 if ((cur_entry->event_enable & async_code) != 0) {
4228                         lock = cur_entry->event_lock;
4229                         if (lock)
4230                                 CAM_SIM_LOCK(path->device->sim);
4231                         cur_entry->callback(cur_entry->callback_arg,
4232                                             async_code, path,
4233                                             async_arg);
4234                         if (lock)
4235                                 CAM_SIM_UNLOCK(path->device->sim);
4236                 }
4237                 cur_entry = next_entry;
4238         }
4239 }
4240
4241 void
4242 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4243 {
4244         union ccb *ccb;
4245         int size;
4246
4247         ccb = xpt_alloc_ccb_nowait();
4248         if (ccb == NULL) {
4249                 xpt_print(path, "Can't allocate CCB to send %s\n",
4250                     xpt_async_string(async_code));
4251                 return;
4252         }
4253
4254         if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4255                 xpt_print(path, "Can't allocate path to send %s\n",
4256                     xpt_async_string(async_code));
4257                 xpt_free_ccb(ccb);
4258                 return;
4259         }
4260         ccb->ccb_h.path->periph = NULL;
4261         ccb->ccb_h.func_code = XPT_ASYNC;
4262         ccb->ccb_h.cbfcnp = xpt_async_process;
4263         ccb->ccb_h.flags |= CAM_UNLOCKED;
4264         ccb->casync.async_code = async_code;
4265         ccb->casync.async_arg_size = 0;
4266         size = xpt_async_size(async_code);
4267         if (size > 0 && async_arg != NULL) {
4268                 ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4269                 if (ccb->casync.async_arg_ptr == NULL) {
4270                         xpt_print(path, "Can't allocate argument to send %s\n",
4271                             xpt_async_string(async_code));
4272                         xpt_free_path(ccb->ccb_h.path);
4273                         xpt_free_ccb(ccb);
4274                         return;
4275                 }
4276                 memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4277                 ccb->casync.async_arg_size = size;
4278         } else if (size < 0) {
4279                 ccb->casync.async_arg_ptr = async_arg;
4280                 ccb->casync.async_arg_size = size;
4281         }
4282         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4283                 xpt_freeze_devq(path, 1);
4284         else
4285                 xpt_freeze_simq(path->bus->sim, 1);
4286         xpt_done(ccb);
4287 }
4288
4289 static void
4290 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4291                       struct cam_et *target, struct cam_ed *device,
4292                       void *async_arg)
4293 {
4294
4295         /*
4296          * We only need to handle events for real devices.
4297          */
4298         if (target->target_id == CAM_TARGET_WILDCARD
4299          || device->lun_id == CAM_LUN_WILDCARD)
4300                 return;
4301
4302         printf("%s called\n", __func__);
4303 }
4304
4305 static uint32_t
4306 xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4307 {
4308         struct cam_devq *devq;
4309         uint32_t freeze;
4310
4311         devq = dev->sim->devq;
4312         mtx_assert(&devq->send_mtx, MA_OWNED);
4313         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4314             ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4315             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4316         freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4317         /* Remove frozen device from sendq. */
4318         if (device_is_queued(dev))
4319                 camq_remove(&devq->send_queue, dev->devq_entry.index);
4320         return (freeze);
4321 }
4322
4323 u_int32_t
4324 xpt_freeze_devq(struct cam_path *path, u_int count)
4325 {
4326         struct cam_ed   *dev = path->device;
4327         struct cam_devq *devq;
4328         uint32_t         freeze;
4329
4330         devq = dev->sim->devq;
4331         mtx_lock(&devq->send_mtx);
4332         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4333         freeze = xpt_freeze_devq_device(dev, count);
4334         mtx_unlock(&devq->send_mtx);
4335         return (freeze);
4336 }
4337
4338 u_int32_t
4339 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4340 {
4341         struct cam_devq *devq;
4342         uint32_t         freeze;
4343
4344         devq = sim->devq;
4345         mtx_lock(&devq->send_mtx);
4346         freeze = (devq->send_queue.qfrozen_cnt += count);
4347         mtx_unlock(&devq->send_mtx);
4348         return (freeze);
4349 }
4350
4351 static void
4352 xpt_release_devq_timeout(void *arg)
4353 {
4354         struct cam_ed *dev;
4355         struct cam_devq *devq;
4356
4357         dev = (struct cam_ed *)arg;
4358         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4359         devq = dev->sim->devq;
4360         mtx_assert(&devq->send_mtx, MA_OWNED);
4361         if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4362                 xpt_run_devq(devq);
4363 }
4364
4365 void
4366 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4367 {
4368         struct cam_ed *dev;
4369         struct cam_devq *devq;
4370
4371         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4372             count, run_queue));
4373         dev = path->device;
4374         devq = dev->sim->devq;
4375         mtx_lock(&devq->send_mtx);
4376         if (xpt_release_devq_device(dev, count, run_queue))
4377                 xpt_run_devq(dev->sim->devq);
4378         mtx_unlock(&devq->send_mtx);
4379 }
4380
4381 static int
4382 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4383 {
4384
4385         mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4386         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4387             ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4388             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4389         if (count > dev->ccbq.queue.qfrozen_cnt) {
4390 #ifdef INVARIANTS
4391                 printf("xpt_release_devq(): requested %u > present %u\n",
4392                     count, dev->ccbq.queue.qfrozen_cnt);
4393 #endif
4394                 count = dev->ccbq.queue.qfrozen_cnt;
4395         }
4396         dev->ccbq.queue.qfrozen_cnt -= count;
4397         if (dev->ccbq.queue.qfrozen_cnt == 0) {
4398                 /*
4399                  * No longer need to wait for a successful
4400                  * command completion.
4401                  */
4402                 dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4403                 /*
4404                  * Remove any timeouts that might be scheduled
4405                  * to release this queue.
4406                  */
4407                 if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4408                         callout_stop(&dev->callout);
4409                         dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4410                 }
4411                 /*
4412                  * Now that we are unfrozen schedule the
4413                  * device so any pending transactions are
4414                  * run.
4415                  */
4416                 xpt_schedule_devq(dev->sim->devq, dev);
4417         } else
4418                 run_queue = 0;
4419         return (run_queue);
4420 }
4421
4422 void
4423 xpt_release_simq(struct cam_sim *sim, int run_queue)
4424 {
4425         struct cam_devq *devq;
4426
4427         devq = sim->devq;
4428         mtx_lock(&devq->send_mtx);
4429         if (devq->send_queue.qfrozen_cnt <= 0) {
4430 #ifdef INVARIANTS
4431                 printf("xpt_release_simq: requested 1 > present %u\n",
4432                     devq->send_queue.qfrozen_cnt);
4433 #endif
4434         } else
4435                 devq->send_queue.qfrozen_cnt--;
4436         if (devq->send_queue.qfrozen_cnt == 0) {
4437                 /*
4438                  * If there is a timeout scheduled to release this
4439                  * sim queue, remove it.  The queue frozen count is
4440                  * already at 0.
4441                  */
4442                 if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4443                         callout_stop(&sim->callout);
4444                         sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4445                 }
4446                 if (run_queue) {
4447                         /*
4448                          * Now that we are unfrozen run the send queue.
4449                          */
4450                         xpt_run_devq(sim->devq);
4451                 }
4452         }
4453         mtx_unlock(&devq->send_mtx);
4454 }
4455
4456 /*
4457  * XXX Appears to be unused.
4458  */
4459 static void
4460 xpt_release_simq_timeout(void *arg)
4461 {
4462         struct cam_sim *sim;
4463
4464         sim = (struct cam_sim *)arg;
4465         xpt_release_simq(sim, /* run_queue */ TRUE);
4466 }
4467
4468 void
4469 xpt_done(union ccb *done_ccb)
4470 {
4471         struct cam_doneq *queue;
4472         int     run, hash;
4473
4474         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4475         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4476                 return;
4477
4478         hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4479             done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4480         queue = &cam_doneqs[hash];
4481         mtx_lock(&queue->cam_doneq_mtx);
4482         run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4483         STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4484         done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4485         mtx_unlock(&queue->cam_doneq_mtx);
4486         if (run)
4487                 wakeup(&queue->cam_doneq);
4488 }
4489
4490 void
4491 xpt_done_direct(union ccb *done_ccb)
4492 {
4493
4494         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4495         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4496                 return;
4497
4498         xpt_done_process(&done_ccb->ccb_h);
4499 }
4500
4501 union ccb *
4502 xpt_alloc_ccb()
4503 {
4504         union ccb *new_ccb;
4505
4506         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4507         return (new_ccb);
4508 }
4509
4510 union ccb *
4511 xpt_alloc_ccb_nowait()
4512 {
4513         union ccb *new_ccb;
4514
4515         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4516         return (new_ccb);
4517 }
4518
4519 void
4520 xpt_free_ccb(union ccb *free_ccb)
4521 {
4522         free(free_ccb, M_CAMCCB);
4523 }
4524
4525
4526
4527 /* Private XPT functions */
4528
4529 /*
4530  * Get a CAM control block for the caller. Charge the structure to the device
4531  * referenced by the path.  If we don't have sufficient resources to allocate
4532  * more ccbs, we return NULL.
4533  */
4534 static union ccb *
4535 xpt_get_ccb_nowait(struct cam_periph *periph)
4536 {
4537         union ccb *new_ccb;
4538
4539         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4540         if (new_ccb == NULL)
4541                 return (NULL);
4542         periph->periph_allocated++;
4543         cam_ccbq_take_opening(&periph->path->device->ccbq);
4544         return (new_ccb);
4545 }
4546
4547 static union ccb *
4548 xpt_get_ccb(struct cam_periph *periph)
4549 {
4550         union ccb *new_ccb;
4551
4552         cam_periph_unlock(periph);
4553         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4554         cam_periph_lock(periph);
4555         periph->periph_allocated++;
4556         cam_ccbq_take_opening(&periph->path->device->ccbq);
4557         return (new_ccb);
4558 }
4559
4560 union ccb *
4561 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4562 {
4563         struct ccb_hdr *ccb_h;
4564
4565         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4566         cam_periph_assert(periph, MA_OWNED);
4567         while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4568             ccb_h->pinfo.priority != priority) {
4569                 if (priority < periph->immediate_priority) {
4570                         periph->immediate_priority = priority;
4571                         xpt_run_allocq(periph, 0);
4572                 } else
4573                         cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4574                             "cgticb", 0);
4575         }
4576         SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4577         return ((union ccb *)ccb_h);
4578 }
4579
4580 static void
4581 xpt_acquire_bus(struct cam_eb *bus)
4582 {
4583
4584         xpt_lock_buses();
4585         bus->refcount++;
4586         xpt_unlock_buses();
4587 }
4588
4589 static void
4590 xpt_release_bus(struct cam_eb *bus)
4591 {
4592
4593         xpt_lock_buses();
4594         KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4595         if (--bus->refcount > 0) {
4596                 xpt_unlock_buses();
4597                 return;
4598         }
4599         TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4600         xsoftc.bus_generation++;
4601         xpt_unlock_buses();
4602         KASSERT(TAILQ_EMPTY(&bus->et_entries),
4603             ("destroying bus, but target list is not empty"));
4604         cam_sim_release(bus->sim);
4605         mtx_destroy(&bus->eb_mtx);
4606         free(bus, M_CAMXPT);
4607 }
4608
4609 static struct cam_et *
4610 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4611 {
4612         struct cam_et *cur_target, *target;
4613
4614         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4615         mtx_assert(&bus->eb_mtx, MA_OWNED);
4616         target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4617                                          M_NOWAIT|M_ZERO);
4618         if (target == NULL)
4619                 return (NULL);
4620
4621         TAILQ_INIT(&target->ed_entries);
4622         target->bus = bus;
4623         target->target_id = target_id;
4624         target->refcount = 1;
4625         target->generation = 0;
4626         target->luns = NULL;
4627         mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4628         timevalclear(&target->last_reset);
4629         /*
4630          * Hold a reference to our parent bus so it
4631          * will not go away before we do.
4632          */
4633         bus->refcount++;
4634
4635         /* Insertion sort into our bus's target list */
4636         cur_target = TAILQ_FIRST(&bus->et_entries);
4637         while (cur_target != NULL && cur_target->target_id < target_id)
4638                 cur_target = TAILQ_NEXT(cur_target, links);
4639         if (cur_target != NULL) {
4640                 TAILQ_INSERT_BEFORE(cur_target, target, links);
4641         } else {
4642                 TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4643         }
4644         bus->generation++;
4645         return (target);
4646 }
4647
4648 static void
4649 xpt_acquire_target(struct cam_et *target)
4650 {
4651         struct cam_eb *bus = target->bus;
4652
4653         mtx_lock(&bus->eb_mtx);
4654         target->refcount++;
4655         mtx_unlock(&bus->eb_mtx);
4656 }
4657
4658 static void
4659 xpt_release_target(struct cam_et *target)
4660 {
4661         struct cam_eb *bus = target->bus;
4662
4663         mtx_lock(&bus->eb_mtx);
4664         if (--target->refcount > 0) {
4665                 mtx_unlock(&bus->eb_mtx);
4666                 return;
4667         }
4668         TAILQ_REMOVE(&bus->et_entries, target, links);
4669         bus->generation++;
4670         mtx_unlock(&bus->eb_mtx);
4671         KASSERT(TAILQ_EMPTY(&target->ed_entries),
4672             ("destroying target, but device list is not empty"));
4673         xpt_release_bus(bus);
4674         mtx_destroy(&target->luns_mtx);
4675         if (target->luns)
4676                 free(target->luns, M_CAMXPT);
4677         free(target, M_CAMXPT);
4678 }
4679
4680 static struct cam_ed *
4681 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4682                          lun_id_t lun_id)
4683 {
4684         struct cam_ed *device;
4685
4686         device = xpt_alloc_device(bus, target, lun_id);
4687         if (device == NULL)
4688                 return (NULL);
4689
4690         device->mintags = 1;
4691         device->maxtags = 1;
4692         return (device);
4693 }
4694
4695 static void
4696 xpt_destroy_device(void *context, int pending)
4697 {
4698         struct cam_ed   *device = context;
4699
4700         mtx_lock(&device->device_mtx);
4701         mtx_destroy(&device->device_mtx);
4702         free(device, M_CAMDEV);
4703 }
4704
4705 struct cam_ed *
4706 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4707 {
4708         struct cam_ed   *cur_device, *device;
4709         struct cam_devq *devq;
4710         cam_status status;
4711
4712         mtx_assert(&bus->eb_mtx, MA_OWNED);
4713         /* Make space for us in the device queue on our bus */
4714         devq = bus->sim->devq;
4715         mtx_lock(&devq->send_mtx);
4716         status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4717         mtx_unlock(&devq->send_mtx);
4718         if (status != CAM_REQ_CMP)
4719                 return (NULL);
4720
4721         device = (struct cam_ed *)malloc(sizeof(*device),
4722                                          M_CAMDEV, M_NOWAIT|M_ZERO);
4723         if (device == NULL)
4724                 return (NULL);
4725
4726         cam_init_pinfo(&device->devq_entry);
4727         device->target = target;
4728         device->lun_id = lun_id;
4729         device->sim = bus->sim;
4730         if (cam_ccbq_init(&device->ccbq,
4731                           bus->sim->max_dev_openings) != 0) {
4732                 free(device, M_CAMDEV);
4733                 return (NULL);
4734         }
4735         SLIST_INIT(&device->asyncs);
4736         SLIST_INIT(&device->periphs);
4737         device->generation = 0;
4738         device->flags = CAM_DEV_UNCONFIGURED;
4739         device->tag_delay_count = 0;
4740         device->tag_saved_openings = 0;
4741         device->refcount = 1;
4742         mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4743         callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4744         TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4745         /*
4746          * Hold a reference to our parent bus so it
4747          * will not go away before we do.
4748          */
4749         target->refcount++;
4750
4751         cur_device = TAILQ_FIRST(&target->ed_entries);
4752         while (cur_device != NULL && cur_device->lun_id < lun_id)
4753                 cur_device = TAILQ_NEXT(cur_device, links);
4754         if (cur_device != NULL)
4755                 TAILQ_INSERT_BEFORE(cur_device, device, links);
4756         else
4757                 TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4758         target->generation++;
4759         return (device);
4760 }
4761
4762 void
4763 xpt_acquire_device(struct cam_ed *device)
4764 {
4765         struct cam_eb *bus = device->target->bus;
4766
4767         mtx_lock(&bus->eb_mtx);
4768         device->refcount++;
4769         mtx_unlock(&bus->eb_mtx);
4770 }
4771
4772 void
4773 xpt_release_device(struct cam_ed *device)
4774 {
4775         struct cam_eb *bus = device->target->bus;
4776         struct cam_devq *devq;
4777
4778         mtx_lock(&bus->eb_mtx);
4779         if (--device->refcount > 0) {
4780                 mtx_unlock(&bus->eb_mtx);
4781                 return;
4782         }
4783
4784         TAILQ_REMOVE(&device->target->ed_entries, device,links);
4785         device->target->generation++;
4786         mtx_unlock(&bus->eb_mtx);
4787
4788         /* Release our slot in the devq */
4789         devq = bus->sim->devq;
4790         mtx_lock(&devq->send_mtx);
4791         cam_devq_resize(devq, devq->send_queue.array_size - 1);
4792         mtx_unlock(&devq->send_mtx);
4793
4794         KASSERT(SLIST_EMPTY(&device->periphs),
4795             ("destroying device, but periphs list is not empty"));
4796         KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4797             ("destroying device while still queued for ccbs"));
4798
4799         if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4800                 callout_stop(&device->callout);
4801
4802         xpt_release_target(device->target);
4803
4804         cam_ccbq_fini(&device->ccbq);
4805         /*
4806          * Free allocated memory.  free(9) does nothing if the
4807          * supplied pointer is NULL, so it is safe to call without
4808          * checking.
4809          */
4810         free(device->supported_vpds, M_CAMXPT);
4811         free(device->device_id, M_CAMXPT);
4812         free(device->ext_inq, M_CAMXPT);
4813         free(device->physpath, M_CAMXPT);
4814         free(device->rcap_buf, M_CAMXPT);
4815         free(device->serial_num, M_CAMXPT);
4816         taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4817 }
4818
4819 u_int32_t
4820 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4821 {
4822         int     result;
4823         struct  cam_ed *dev;
4824
4825         dev = path->device;
4826         mtx_lock(&dev->sim->devq->send_mtx);
4827         result = cam_ccbq_resize(&dev->ccbq, newopenings);
4828         mtx_unlock(&dev->sim->devq->send_mtx);
4829         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4830          || (dev->inq_flags & SID_CmdQue) != 0)
4831                 dev->tag_saved_openings = newopenings;
4832         return (result);
4833 }
4834
4835 static struct cam_eb *
4836 xpt_find_bus(path_id_t path_id)
4837 {
4838         struct cam_eb *bus;
4839
4840         xpt_lock_buses();
4841         for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4842              bus != NULL;
4843              bus = TAILQ_NEXT(bus, links)) {
4844                 if (bus->path_id == path_id) {
4845                         bus->refcount++;
4846                         break;
4847                 }
4848         }
4849         xpt_unlock_buses();
4850         return (bus);
4851 }
4852
4853 static struct cam_et *
4854 xpt_find_target(struct cam_eb *bus, target_id_t target_id)
4855 {
4856         struct cam_et *target;
4857
4858         mtx_assert(&bus->eb_mtx, MA_OWNED);
4859         for (target = TAILQ_FIRST(&bus->et_entries);
4860              target != NULL;
4861              target = TAILQ_NEXT(target, links)) {
4862                 if (target->target_id == target_id) {
4863                         target->refcount++;
4864                         break;
4865                 }
4866         }
4867         return (target);
4868 }
4869
4870 static struct cam_ed *
4871 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4872 {
4873         struct cam_ed *device;
4874
4875         mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4876         for (device = TAILQ_FIRST(&target->ed_entries);
4877              device != NULL;
4878              device = TAILQ_NEXT(device, links)) {
4879                 if (device->lun_id == lun_id) {
4880                         device->refcount++;
4881                         break;
4882                 }
4883         }
4884         return (device);
4885 }
4886
4887 void
4888 xpt_start_tags(struct cam_path *path)
4889 {
4890         struct ccb_relsim crs;
4891         struct cam_ed *device;
4892         struct cam_sim *sim;
4893         int    newopenings;
4894
4895         device = path->device;
4896         sim = path->bus->sim;
4897         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4898         xpt_freeze_devq(path, /*count*/1);
4899         device->inq_flags |= SID_CmdQue;
4900         if (device->tag_saved_openings != 0)
4901                 newopenings = device->tag_saved_openings;
4902         else
4903                 newopenings = min(device->maxtags,
4904                                   sim->max_tagged_dev_openings);
4905         xpt_dev_ccbq_resize(path, newopenings);
4906         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4907         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4908         crs.ccb_h.func_code = XPT_REL_SIMQ;
4909         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4910         crs.openings
4911             = crs.release_timeout
4912             = crs.qfrozen_cnt
4913             = 0;
4914         xpt_action((union ccb *)&crs);
4915 }
4916
4917 void
4918 xpt_stop_tags(struct cam_path *path)
4919 {
4920         struct ccb_relsim crs;
4921         struct cam_ed *device;
4922         struct cam_sim *sim;
4923
4924         device = path->device;
4925         sim = path->bus->sim;
4926         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4927         device->tag_delay_count = 0;
4928         xpt_freeze_devq(path, /*count*/1);
4929         device->inq_flags &= ~SID_CmdQue;
4930         xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4931         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4932         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4933         crs.ccb_h.func_code = XPT_REL_SIMQ;
4934         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4935         crs.openings
4936             = crs.release_timeout
4937             = crs.qfrozen_cnt
4938             = 0;
4939         xpt_action((union ccb *)&crs);
4940 }
4941
4942 static void
4943 xpt_boot_delay(void *arg)
4944 {
4945
4946         xpt_release_boot();
4947 }
4948
4949 static void
4950 xpt_config(void *arg)
4951 {
4952         /*
4953          * Now that interrupts are enabled, go find our devices
4954          */
4955         if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4956                 printf("xpt_config: failed to create taskqueue thread.\n");
4957
4958         /* Setup debugging path */
4959         if (cam_dflags != CAM_DEBUG_NONE) {
4960                 if (xpt_create_path(&cam_dpath, NULL,
4961                                     CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4962                                     CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4963                         printf("xpt_config: xpt_create_path() failed for debug"
4964                                " target %d:%d:%d, debugging disabled\n",
4965                                CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4966                         cam_dflags = CAM_DEBUG_NONE;
4967                 }
4968         } else
4969                 cam_dpath = NULL;
4970
4971         periphdriver_init(1);
4972         xpt_hold_boot();
4973         callout_init(&xsoftc.boot_callout, 1);
4974         callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
4975             xpt_boot_delay, NULL, 0);
4976         /* Fire up rescan thread. */
4977         if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4978             "cam", "scanner")) {
4979                 printf("xpt_config: failed to create rescan thread.\n");
4980         }
4981 }
4982
4983 void
4984 xpt_hold_boot(void)
4985 {
4986         xpt_lock_buses();
4987         xsoftc.buses_to_config++;
4988         xpt_unlock_buses();
4989 }
4990
4991 void
4992 xpt_release_boot(void)
4993 {
4994         xpt_lock_buses();
4995         xsoftc.buses_to_config--;
4996         if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4997                 struct  xpt_task *task;
4998
4999                 xsoftc.buses_config_done = 1;
5000                 xpt_unlock_buses();
5001                 /* Call manually because we don't have any busses */
5002                 task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
5003                 if (task != NULL) {
5004                         TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
5005                         taskqueue_enqueue(taskqueue_thread, &task->task);
5006                 }
5007         } else
5008                 xpt_unlock_buses();
5009 }
5010
5011 /*
5012  * If the given device only has one peripheral attached to it, and if that
5013  * peripheral is the passthrough driver, announce it.  This insures that the
5014  * user sees some sort of announcement for every peripheral in their system.
5015  */
5016 static int
5017 xptpassannouncefunc(struct cam_ed *device, void *arg)
5018 {
5019         struct cam_periph *periph;
5020         int i;
5021
5022         for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5023              periph = SLIST_NEXT(periph, periph_links), i++);
5024
5025         periph = SLIST_FIRST(&device->periphs);
5026         if ((i == 1)
5027          && (strncmp(periph->periph_name, "pass", 4) == 0))
5028                 xpt_announce_periph(periph, NULL);
5029
5030         return(1);
5031 }
5032
5033 static void
5034 xpt_finishconfig_task(void *context, int pending)
5035 {
5036
5037         periphdriver_init(2);
5038         /*
5039          * Check for devices with no "standard" peripheral driver
5040          * attached.  For any devices like that, announce the
5041          * passthrough driver so the user will see something.
5042          */
5043         if (!bootverbose)
5044                 xpt_for_all_devices(xptpassannouncefunc, NULL);
5045
5046         /* Release our hook so that the boot can continue. */
5047         config_intrhook_disestablish(xsoftc.xpt_config_hook);
5048         free(xsoftc.xpt_config_hook, M_CAMXPT);
5049         xsoftc.xpt_config_hook = NULL;
5050
5051         free(context, M_CAMXPT);
5052 }
5053
5054 cam_status
5055 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5056                    struct cam_path *path)
5057 {
5058         struct ccb_setasync csa;
5059         cam_status status;
5060         int xptpath = 0;
5061
5062         if (path == NULL) {
5063                 status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5064                                          CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5065                 if (status != CAM_REQ_CMP)
5066                         return (status);
5067                 xpt_path_lock(path);
5068                 xptpath = 1;
5069         }
5070
5071         xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5072         csa.ccb_h.func_code = XPT_SASYNC_CB;
5073         csa.event_enable = event;
5074         csa.callback = cbfunc;
5075         csa.callback_arg = cbarg;
5076         xpt_action((union ccb *)&csa);
5077         status = csa.ccb_h.status;
5078
5079         if (xptpath) {
5080                 xpt_path_unlock(path);
5081                 xpt_free_path(path);
5082         }
5083
5084         if ((status == CAM_REQ_CMP) &&
5085             (csa.event_enable & AC_FOUND_DEVICE)) {
5086                 /*
5087                  * Get this peripheral up to date with all
5088                  * the currently existing devices.
5089                  */
5090                 xpt_for_all_devices(xptsetasyncfunc, &csa);
5091         }
5092         if ((status == CAM_REQ_CMP) &&
5093             (csa.event_enable & AC_PATH_REGISTERED)) {
5094                 /*
5095                  * Get this peripheral up to date with all
5096                  * the currently existing busses.
5097                  */
5098                 xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5099         }
5100
5101         return (status);
5102 }
5103
5104 static void
5105 xptaction(struct cam_sim *sim, union ccb *work_ccb)
5106 {
5107         CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5108
5109         switch (work_ccb->ccb_h.func_code) {
5110         /* Common cases first */
5111         case XPT_PATH_INQ:              /* Path routing inquiry */
5112         {
5113                 struct ccb_pathinq *cpi;
5114
5115                 cpi = &work_ccb->cpi;
5116                 cpi->version_num = 1; /* XXX??? */
5117                 cpi->hba_inquiry = 0;
5118                 cpi->target_sprt = 0;
5119                 cpi->hba_misc = 0;
5120                 cpi->hba_eng_cnt = 0;
5121                 cpi->max_target = 0;
5122                 cpi->max_lun = 0;
5123                 cpi->initiator_id = 0;
5124                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5125                 strncpy(cpi->hba_vid, "", HBA_IDLEN);
5126                 strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5127                 cpi->unit_number = sim->unit_number;
5128                 cpi->bus_id = sim->bus_id;
5129                 cpi->base_transfer_speed = 0;
5130                 cpi->protocol = PROTO_UNSPECIFIED;
5131                 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5132                 cpi->transport = XPORT_UNSPECIFIED;
5133                 cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5134                 cpi->ccb_h.status = CAM_REQ_CMP;
5135                 xpt_done(work_ccb);
5136                 break;
5137         }
5138         default:
5139                 work_ccb->ccb_h.status = CAM_REQ_INVALID;
5140                 xpt_done(work_ccb);
5141                 break;
5142         }
5143 }
5144
5145 /*
5146  * The xpt as a "controller" has no interrupt sources, so polling
5147  * is a no-op.
5148  */
5149 static void
5150 xptpoll(struct cam_sim *sim)
5151 {
5152 }
5153
5154 void
5155 xpt_lock_buses(void)
5156 {
5157         mtx_lock(&xsoftc.xpt_topo_lock);
5158 }
5159
5160 void
5161 xpt_unlock_buses(void)
5162 {
5163         mtx_unlock(&xsoftc.xpt_topo_lock);
5164 }
5165
5166 struct mtx *
5167 xpt_path_mtx(struct cam_path *path)
5168 {
5169
5170         return (&path->device->device_mtx);
5171 }
5172
5173 static void
5174 xpt_done_process(struct ccb_hdr *ccb_h)
5175 {
5176         struct cam_sim *sim;
5177         struct cam_devq *devq;
5178         struct mtx *mtx = NULL;
5179
5180         if (ccb_h->flags & CAM_HIGH_POWER) {
5181                 struct highpowerlist    *hphead;
5182                 struct cam_ed           *device;
5183
5184                 mtx_lock(&xsoftc.xpt_highpower_lock);
5185                 hphead = &xsoftc.highpowerq;
5186
5187                 device = STAILQ_FIRST(hphead);
5188
5189                 /*
5190                  * Increment the count since this command is done.
5191                  */
5192                 xsoftc.num_highpower++;
5193
5194                 /*
5195                  * Any high powered commands queued up?
5196                  */
5197                 if (device != NULL) {
5198
5199                         STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5200                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5201
5202                         mtx_lock(&device->sim->devq->send_mtx);
5203                         xpt_release_devq_device(device,
5204                                          /*count*/1, /*runqueue*/TRUE);
5205                         mtx_unlock(&device->sim->devq->send_mtx);
5206                 } else
5207                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5208         }
5209
5210         sim = ccb_h->path->bus->sim;
5211
5212         if (ccb_h->status & CAM_RELEASE_SIMQ) {
5213                 xpt_release_simq(sim, /*run_queue*/FALSE);
5214                 ccb_h->status &= ~CAM_RELEASE_SIMQ;
5215         }
5216
5217         if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5218          && (ccb_h->status & CAM_DEV_QFRZN)) {
5219                 xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5220                 ccb_h->status &= ~CAM_DEV_QFRZN;
5221         }
5222
5223         devq = sim->devq;
5224         if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5225                 struct cam_ed *dev = ccb_h->path->device;
5226
5227                 mtx_lock(&devq->send_mtx);
5228                 devq->send_active--;
5229                 devq->send_openings++;
5230                 cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5231
5232                 if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5233                   && (dev->ccbq.dev_active == 0))) {
5234                         dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5235                         xpt_release_devq_device(dev, /*count*/1,
5236                                          /*run_queue*/FALSE);
5237                 }
5238
5239                 if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5240                   && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5241                         dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5242                         xpt_release_devq_device(dev, /*count*/1,
5243                                          /*run_queue*/FALSE);
5244                 }
5245
5246                 if (!device_is_queued(dev))
5247                         (void)xpt_schedule_devq(devq, dev);
5248                 xpt_run_devq(devq);
5249                 mtx_unlock(&devq->send_mtx);
5250
5251                 if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5252                         mtx = xpt_path_mtx(ccb_h->path);
5253                         mtx_lock(mtx);
5254
5255                         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5256                          && (--dev->tag_delay_count == 0))
5257                                 xpt_start_tags(ccb_h->path);
5258                 }
5259         }
5260
5261         if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5262                 if (mtx == NULL) {
5263                         mtx = xpt_path_mtx(ccb_h->path);
5264                         mtx_lock(mtx);
5265                 }
5266         } else {
5267                 if (mtx != NULL) {
5268                         mtx_unlock(mtx);
5269                         mtx = NULL;
5270                 }
5271         }
5272
5273         /* Call the peripheral driver's callback */
5274         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5275         (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5276         if (mtx != NULL)
5277                 mtx_unlock(mtx);
5278 }
5279
5280 void
5281 xpt_done_td(void *arg)
5282 {
5283         struct cam_doneq *queue = arg;
5284         struct ccb_hdr *ccb_h;
5285         STAILQ_HEAD(, ccb_hdr)  doneq;
5286
5287         STAILQ_INIT(&doneq);
5288         mtx_lock(&queue->cam_doneq_mtx);
5289         while (1) {
5290                 while (STAILQ_EMPTY(&queue->cam_doneq)) {
5291                         queue->cam_doneq_sleep = 1;
5292                         msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5293                             PRIBIO, "-", 0);
5294                         queue->cam_doneq_sleep = 0;
5295                 }
5296                 STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5297                 mtx_unlock(&queue->cam_doneq_mtx);
5298
5299                 THREAD_NO_SLEEPING();
5300                 while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5301                         STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5302                         xpt_done_process(ccb_h);
5303                 }
5304                 THREAD_SLEEPING_OK();
5305
5306                 mtx_lock(&queue->cam_doneq_mtx);
5307         }
5308 }
5309
5310 static void
5311 camisr_runqueue(void)
5312 {
5313         struct  ccb_hdr *ccb_h;
5314         struct cam_doneq *queue;
5315         int i;
5316
5317         /* Process global queues. */
5318         for (i = 0; i < cam_num_doneqs; i++) {
5319                 queue = &cam_doneqs[i];
5320                 mtx_lock(&queue->cam_doneq_mtx);
5321                 while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5322                         STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5323                         mtx_unlock(&queue->cam_doneq_mtx);
5324                         xpt_done_process(ccb_h);
5325                         mtx_lock(&queue->cam_doneq_mtx);
5326                 }
5327                 mtx_unlock(&queue->cam_doneq_mtx);
5328         }
5329 }