sys/cam/cam_xpt.c

   1 /*-
   2  * Implementation of the Common Access Method Transport (XPT) layer.
   3  *
   4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
   5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification, immediately at the beginning of the file.
  14  * 2. The name of the author may not be used to endorse or promote products
  15  *    derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/bus.h>
  35 #include <sys/systm.h>
  36 #include <sys/types.h>
  37 #include <sys/malloc.h>
  38 #include <sys/kernel.h>
  39 #include <sys/time.h>
  40 #include <sys/conf.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/interrupt.h>
  43 #include <sys/proc.h>
  44 #include <sys/sbuf.h>
  45 #include <sys/smp.h>
  46 #include <sys/taskqueue.h>
  47
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/sysctl.h>
  51 #include <sys/kthread.h>
  52
  53 #include <cam/cam.h>
  54 #include <cam/cam_ccb.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/cam_queue.h>
  57 #include <cam/cam_sim.h>
  58 #include <cam/cam_xpt.h>
  59 #include <cam/cam_xpt_sim.h>
  60 #include <cam/cam_xpt_periph.h>
  61 #include <cam/cam_xpt_internal.h>
  62 #include <cam/cam_debug.h>
  63 #include <cam/cam_compat.h>
  64
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_message.h>
  67 #include <cam/scsi/scsi_pass.h>
  68
  69 #include <machine/md_var.h>     /* geometry translation */
  70 #include <machine/stdarg.h>     /* for xpt_print below */
  71
  72 #include "opt_cam.h"
  73
  74 /*
  75  * This is the maximum number of high powered commands (e.g. start unit)
  76  * that can be outstanding at a particular time.
  77  */
  78 #ifndef CAM_MAX_HIGHPOWER
  79 #define CAM_MAX_HIGHPOWER  4
  80 #endif
  81
  82 /* Datastructures internal to the xpt layer */
  83 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
  84 MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
  85 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
  86 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
  87
  88 /* Object for defering XPT actions to a taskqueue */
  89 struct xpt_task {
  90         struct task     task;
  91         void            *data1;
  92         uintptr_t       data2;
  93 };
  94
  95 struct xpt_softc {
  96         /* number of high powered commands that can go through right now */
  97         struct mtx              xpt_highpower_lock;
  98         STAILQ_HEAD(highpowerlist, cam_ed)      highpowerq;
  99         int                     num_highpower;
 100
 101         /* queue for handling async rescan requests. */
 102         TAILQ_HEAD(, ccb_hdr) ccb_scanq;
 103         int buses_to_config;
 104         int buses_config_done;
 105
 106         /* Registered busses */
 107         TAILQ_HEAD(,cam_eb)     xpt_busses;
 108         u_int                   bus_generation;
 109
 110         struct intr_config_hook *xpt_config_hook;
 111
 112         int                     boot_delay;
 113         struct callout          boot_callout;
 114
 115         struct mtx              xpt_topo_lock;
 116         struct mtx              xpt_lock;
 117         struct taskqueue        *xpt_taskq;
 118 };
 119
 120 typedef enum {
 121         DM_RET_COPY             = 0x01,
 122         DM_RET_FLAG_MASK        = 0x0f,
 123         DM_RET_NONE             = 0x00,
 124         DM_RET_STOP             = 0x10,
 125         DM_RET_DESCEND          = 0x20,
 126         DM_RET_ERROR            = 0x30,
 127         DM_RET_ACTION_MASK      = 0xf0
 128 } dev_match_ret;
 129
 130 typedef enum {
 131         XPT_DEPTH_BUS,
 132         XPT_DEPTH_TARGET,
 133         XPT_DEPTH_DEVICE,
 134         XPT_DEPTH_PERIPH
 135 } xpt_traverse_depth;
 136
 137 struct xpt_traverse_config {
 138         xpt_traverse_depth      depth;
 139         void                    *tr_func;
 140         void                    *tr_arg;
 141 };
 142
 143 typedef int     xpt_busfunc_t (struct cam_eb *bus, void *arg);
 144 typedef int     xpt_targetfunc_t (struct cam_et *target, void *arg);
 145 typedef int     xpt_devicefunc_t (struct cam_ed *device, void *arg);
 146 typedef int     xpt_periphfunc_t (struct cam_periph *periph, void *arg);
 147 typedef int     xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
 148
 149 /* Transport layer configuration information */
 150 static struct xpt_softc xsoftc;
 151
 152 TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
 153 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
 154            &xsoftc.boot_delay, 0, "Bus registration wait time");
 155
 156 struct cam_doneq {
 157         struct mtx_padalign     cam_doneq_mtx;
 158         STAILQ_HEAD(, ccb_hdr)  cam_doneq;
 159         int                     cam_doneq_sleep;
 160 };
 161
 162 static struct cam_doneq cam_doneqs[MAXCPU];
 163 static int cam_num_doneqs;
 164 static struct proc *cam_proc;
 165
 166 TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs);
 167 SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
 168            &cam_num_doneqs, 0, "Number of completion queues/threads");
 169
 170 struct cam_periph *xpt_periph;
 171
 172 static periph_init_t xpt_periph_init;
 173
 174 static struct periph_driver xpt_driver =
 175 {
 176         xpt_periph_init, "xpt",
 177         TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
 178         CAM_PERIPH_DRV_EARLY
 179 };
 180
 181 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
 182
 183 static d_open_t xptopen;
 184 static d_close_t xptclose;
 185 static d_ioctl_t xptioctl;
 186 static d_ioctl_t xptdoioctl;
 187
 188 static struct cdevsw xpt_cdevsw = {
 189         .d_version =    D_VERSION,
 190         .d_flags =      0,
 191         .d_open =       xptopen,
 192         .d_close =      xptclose,
 193         .d_ioctl =      xptioctl,
 194         .d_name =       "xpt",
 195 };
 196
 197 /* Storage for debugging datastructures */
 198 struct cam_path *cam_dpath;
 199 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
 200 TUNABLE_INT("kern.cam.dflags", &cam_dflags);
 201 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
 202         &cam_dflags, 0, "Enabled debug flags");
 203 u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
 204 TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
 205 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
 206         &cam_debug_delay, 0, "Delay in us after each debug message");
 207
 208 /* Our boot-time initialization hook */
 209 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 210
 211 static moduledata_t cam_moduledata = {
 212         "cam",
 213         cam_module_event_handler,
 214         NULL
 215 };
 216
 217 static int      xpt_init(void *);
 218
 219 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
 220 MODULE_VERSION(cam, 1);
 221
 222
 223 static void             xpt_async_bcast(struct async_list *async_head,
 224                                         u_int32_t async_code,
 225                                         struct cam_path *path,
 226                                         void *async_arg);
 227 static path_id_t xptnextfreepathid(void);
 228 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
 229 static union ccb *xpt_get_ccb(struct cam_periph *periph);
 230 static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
 231 static void      xpt_run_allocq(struct cam_periph *periph, int sleep);
 232 static void      xpt_run_allocq_task(void *context, int pending);
 233 static void      xpt_run_devq(struct cam_devq *devq);
 234 static timeout_t xpt_release_devq_timeout;
 235 static void      xpt_release_simq_timeout(void *arg) __unused;
 236 static void      xpt_acquire_bus(struct cam_eb *bus);
 237 static void      xpt_release_bus(struct cam_eb *bus);
 238 static uint32_t  xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
 239 static int       xpt_release_devq_device(struct cam_ed *dev, u_int count,
 240                     int run_queue);
 241 static struct cam_et*
 242                  xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
 243 static void      xpt_acquire_target(struct cam_et *target);
 244 static void      xpt_release_target(struct cam_et *target);
 245 static struct cam_eb*
 246                  xpt_find_bus(path_id_t path_id);
 247 static struct cam_et*
 248                  xpt_find_target(struct cam_eb *bus, target_id_t target_id);
 249 static struct cam_ed*
 250                  xpt_find_device(struct cam_et *target, lun_id_t lun_id);
 251 static void      xpt_config(void *arg);
 252 static int       xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 253                                  u_int32_t new_priority);
 254 static xpt_devicefunc_t xptpassannouncefunc;
 255 static void      xptaction(struct cam_sim *sim, union ccb *work_ccb);
 256 static void      xptpoll(struct cam_sim *sim);
 257 static void      camisr_runqueue(void);
 258 static void      xpt_done_process(struct ccb_hdr *ccb_h);
 259 static void      xpt_done_td(void *);
 260 static dev_match_ret    xptbusmatch(struct dev_match_pattern *patterns,
 261                                     u_int num_patterns, struct cam_eb *bus);
 262 static dev_match_ret    xptdevicematch(struct dev_match_pattern *patterns,
 263                                        u_int num_patterns,
 264                                        struct cam_ed *device);
 265 static dev_match_ret    xptperiphmatch(struct dev_match_pattern *patterns,
 266                                        u_int num_patterns,
 267                                        struct cam_periph *periph);
 268 static xpt_busfunc_t    xptedtbusfunc;
 269 static xpt_targetfunc_t xptedttargetfunc;
 270 static xpt_devicefunc_t xptedtdevicefunc;
 271 static xpt_periphfunc_t xptedtperiphfunc;
 272 static xpt_pdrvfunc_t   xptplistpdrvfunc;
 273 static xpt_periphfunc_t xptplistperiphfunc;
 274 static int              xptedtmatch(struct ccb_dev_match *cdm);
 275 static int              xptperiphlistmatch(struct ccb_dev_match *cdm);
 276 static int              xptbustraverse(struct cam_eb *start_bus,
 277                                        xpt_busfunc_t *tr_func, void *arg);
 278 static int              xpttargettraverse(struct cam_eb *bus,
 279                                           struct cam_et *start_target,
 280                                           xpt_targetfunc_t *tr_func, void *arg);
 281 static int              xptdevicetraverse(struct cam_et *target,
 282                                           struct cam_ed *start_device,
 283                                           xpt_devicefunc_t *tr_func, void *arg);
 284 static int              xptperiphtraverse(struct cam_ed *device,
 285                                           struct cam_periph *start_periph,
 286                                           xpt_periphfunc_t *tr_func, void *arg);
 287 static int              xptpdrvtraverse(struct periph_driver **start_pdrv,
 288                                         xpt_pdrvfunc_t *tr_func, void *arg);
 289 static int              xptpdperiphtraverse(struct periph_driver **pdrv,
 290                                             struct cam_periph *start_periph,
 291                                             xpt_periphfunc_t *tr_func,
 292                                             void *arg);
 293 static xpt_busfunc_t    xptdefbusfunc;
 294 static xpt_targetfunc_t xptdeftargetfunc;
 295 static xpt_devicefunc_t xptdefdevicefunc;
 296 static xpt_periphfunc_t xptdefperiphfunc;
 297 static void             xpt_finishconfig_task(void *context, int pending);
 298 static void             xpt_dev_async_default(u_int32_t async_code,
 299                                               struct cam_eb *bus,
 300                                               struct cam_et *target,
 301                                               struct cam_ed *device,
 302                                               void *async_arg);
 303 static struct cam_ed *  xpt_alloc_device_default(struct cam_eb *bus,
 304                                                  struct cam_et *target,
 305                                                  lun_id_t lun_id);
 306 static xpt_devicefunc_t xptsetasyncfunc;
 307 static xpt_busfunc_t    xptsetasyncbusfunc;
 308 static cam_status       xptregister(struct cam_periph *periph,
 309                                     void *arg);
 310 static __inline int device_is_queued(struct cam_ed *device);
 311
 312 static __inline int
 313 xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
 314 {
 315         int     retval;
 316
 317         mtx_assert(&devq->send_mtx, MA_OWNED);
 318         if ((dev->ccbq.queue.entries > 0) &&
 319             (dev->ccbq.dev_openings > 0) &&
 320             (dev->ccbq.queue.qfrozen_cnt == 0)) {
 321                 /*
 322                  * The priority of a device waiting for controller
 323                  * resources is that of the highest priority CCB
 324                  * enqueued.
 325                  */
 326                 retval =
 327                     xpt_schedule_dev(&devq->send_queue,
 328                                      &dev->devq_entry,
 329                                      CAMQ_GET_PRIO(&dev->ccbq.queue));
 330         } else {
 331                 retval = 0;
 332         }
 333         return (retval);
 334 }
 335
 336 static __inline int
 337 device_is_queued(struct cam_ed *device)
 338 {
 339         return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
 340 }
 341
 342 static void
 343 xpt_periph_init()
 344 {
 345         make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
 346 }
 347
 348 static int
 349 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 350 {
 351
 352         /*
 353          * Only allow read-write access.
 354          */
 355         if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
 356                 return(EPERM);
 357
 358         /*
 359          * We don't allow nonblocking access.
 360          */
 361         if ((flags & O_NONBLOCK) != 0) {
 362                 printf("%s: can't do nonblocking access\n", devtoname(dev));
 363                 return(ENODEV);
 364         }
 365
 366         return(0);
 367 }
 368
 369 static int
 370 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 371 {
 372
 373         return(0);
 374 }
 375
 376 /*
 377  * Don't automatically grab the xpt softc lock here even though this is going
 378  * through the xpt device.  The xpt device is really just a back door for
 379  * accessing other devices and SIMs, so the right thing to do is to grab
 380  * the appropriate SIM lock once the bus/SIM is located.
 381  */
 382 static int
 383 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 384 {
 385         int error;
 386
 387         if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
 388                 error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
 389         }
 390         return (error);
 391 }
 392
 393 static int
 394 xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 395 {
 396         int error;
 397
 398         error = 0;
 399
 400         switch(cmd) {
 401         /*
 402          * For the transport layer CAMIOCOMMAND ioctl, we really only want
 403          * to accept CCB types that don't quite make sense to send through a
 404          * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
 405          * in the CAM spec.
 406          */
 407         case CAMIOCOMMAND: {
 408                 union ccb *ccb;
 409                 union ccb *inccb;
 410                 struct cam_eb *bus;
 411
 412                 inccb = (union ccb *)addr;
 413
 414                 bus = xpt_find_bus(inccb->ccb_h.path_id);
 415                 if (bus == NULL)
 416                         return (EINVAL);
 417
 418                 switch (inccb->ccb_h.func_code) {
 419                 case XPT_SCAN_BUS:
 420                 case XPT_RESET_BUS:
 421                         if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
 422                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 423                                 xpt_release_bus(bus);
 424                                 return (EINVAL);
 425                         }
 426                         break;
 427                 case XPT_SCAN_TGT:
 428                         if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
 429                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 430                                 xpt_release_bus(bus);
 431                                 return (EINVAL);
 432                         }
 433                         break;
 434                 default:
 435                         break;
 436                 }
 437
 438                 switch(inccb->ccb_h.func_code) {
 439                 case XPT_SCAN_BUS:
 440                 case XPT_RESET_BUS:
 441                 case XPT_PATH_INQ:
 442                 case XPT_ENG_INQ:
 443                 case XPT_SCAN_LUN:
 444                 case XPT_SCAN_TGT:
 445
 446                         ccb = xpt_alloc_ccb();
 447
 448                         /*
 449                          * Create a path using the bus, target, and lun the
 450                          * user passed in.
 451                          */
 452                         if (xpt_create_path(&ccb->ccb_h.path, NULL,
 453                                             inccb->ccb_h.path_id,
 454                                             inccb->ccb_h.target_id,
 455                                             inccb->ccb_h.target_lun) !=
 456                                             CAM_REQ_CMP){
 457                                 error = EINVAL;
 458                                 xpt_free_ccb(ccb);
 459                                 break;
 460                         }
 461                         /* Ensure all of our fields are correct */
 462                         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
 463                                       inccb->ccb_h.pinfo.priority);
 464                         xpt_merge_ccb(ccb, inccb);
 465                         xpt_path_lock(ccb->ccb_h.path);
 466                         cam_periph_runccb(ccb, NULL, 0, 0, NULL);
 467                         xpt_path_unlock(ccb->ccb_h.path);
 468                         bcopy(ccb, inccb, sizeof(union ccb));
 469                         xpt_free_path(ccb->ccb_h.path);
 470                         xpt_free_ccb(ccb);
 471                         break;
 472
 473                 case XPT_DEBUG: {
 474                         union ccb ccb;
 475
 476                         /*
 477                          * This is an immediate CCB, so it's okay to
 478                          * allocate it on the stack.
 479                          */
 480
 481                         /*
 482                          * Create a path using the bus, target, and lun the
 483                          * user passed in.
 484                          */
 485                         if (xpt_create_path(&ccb.ccb_h.path, NULL,
 486                                             inccb->ccb_h.path_id,
 487                                             inccb->ccb_h.target_id,
 488                                             inccb->ccb_h.target_lun) !=
 489                                             CAM_REQ_CMP){
 490                                 error = EINVAL;
 491                                 break;
 492                         }
 493                         /* Ensure all of our fields are correct */
 494                         xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
 495                                       inccb->ccb_h.pinfo.priority);
 496                         xpt_merge_ccb(&ccb, inccb);
 497                         xpt_action(&ccb);
 498                         bcopy(&ccb, inccb, sizeof(union ccb));
 499                         xpt_free_path(ccb.ccb_h.path);
 500                         break;
 501
 502                 }
 503                 case XPT_DEV_MATCH: {
 504                         struct cam_periph_map_info mapinfo;
 505                         struct cam_path *old_path;
 506
 507                         /*
 508                          * We can't deal with physical addresses for this
 509                          * type of transaction.
 510                          */
 511                         if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
 512                             CAM_DATA_VADDR) {
 513                                 error = EINVAL;
 514                                 break;
 515                         }
 516
 517                         /*
 518                          * Save this in case the caller had it set to
 519                          * something in particular.
 520                          */
 521                         old_path = inccb->ccb_h.path;
 522
 523                         /*
 524                          * We really don't need a path for the matching
 525                          * code.  The path is needed because of the
 526                          * debugging statements in xpt_action().  They
 527                          * assume that the CCB has a valid path.
 528                          */
 529                         inccb->ccb_h.path = xpt_periph->path;
 530
 531                         bzero(&mapinfo, sizeof(mapinfo));
 532
 533                         /*
 534                          * Map the pattern and match buffers into kernel
 535                          * virtual address space.
 536                          */
 537                         error = cam_periph_mapmem(inccb, &mapinfo);
 538
 539                         if (error) {
 540                                 inccb->ccb_h.path = old_path;
 541                                 break;
 542                         }
 543
 544                         /*
 545                          * This is an immediate CCB, we can send it on directly.
 546                          */
 547                         xpt_action(inccb);
 548
 549                         /*
 550                          * Map the buffers back into user space.
 551                          */
 552                         cam_periph_unmapmem(inccb, &mapinfo);
 553
 554                         inccb->ccb_h.path = old_path;
 555
 556                         error = 0;
 557                         break;
 558                 }
 559                 default:
 560                         error = ENOTSUP;
 561                         break;
 562                 }
 563                 xpt_release_bus(bus);
 564                 break;
 565         }
 566         /*
 567          * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
 568          * with the periphal driver name and unit name filled in.  The other
 569          * fields don't really matter as input.  The passthrough driver name
 570          * ("pass"), and unit number are passed back in the ccb.  The current
 571          * device generation number, and the index into the device peripheral
 572          * driver list, and the status are also passed back.  Note that
 573          * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
 574          * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
 575          * (or rather should be) impossible for the device peripheral driver
 576          * list to change since we look at the whole thing in one pass, and
 577          * we do it with lock protection.
 578          *
 579          */
 580         case CAMGETPASSTHRU: {
 581                 union ccb *ccb;
 582                 struct cam_periph *periph;
 583                 struct periph_driver **p_drv;
 584                 char   *name;
 585                 u_int unit;
 586                 int base_periph_found;
 587
 588                 ccb = (union ccb *)addr;
 589                 unit = ccb->cgdl.unit_number;
 590                 name = ccb->cgdl.periph_name;
 591                 base_periph_found = 0;
 592
 593                 /*
 594                  * Sanity check -- make sure we don't get a null peripheral
 595                  * driver name.
 596                  */
 597                 if (*ccb->cgdl.periph_name == '\0') {
 598                         error = EINVAL;
 599                         break;
 600                 }
 601
 602                 /* Keep the list from changing while we traverse it */
 603                 xpt_lock_buses();
 604
 605                 /* first find our driver in the list of drivers */
 606                 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
 607                         if (strcmp((*p_drv)->driver_name, name) == 0)
 608                                 break;
 609
 610                 if (*p_drv == NULL) {
 611                         xpt_unlock_buses();
 612                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 613                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 614                         *ccb->cgdl.periph_name = '\0';
 615                         ccb->cgdl.unit_number = 0;
 616                         error = ENOENT;
 617                         break;
 618                 }
 619
 620                 /*
 621                  * Run through every peripheral instance of this driver
 622                  * and check to see whether it matches the unit passed
 623                  * in by the user.  If it does, get out of the loops and
 624                  * find the passthrough driver associated with that
 625                  * peripheral driver.
 626                  */
 627                 for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
 628                      periph = TAILQ_NEXT(periph, unit_links)) {
 629
 630                         if (periph->unit_number == unit)
 631                                 break;
 632                 }
 633                 /*
 634                  * If we found the peripheral driver that the user passed
 635                  * in, go through all of the peripheral drivers for that
 636                  * particular device and look for a passthrough driver.
 637                  */
 638                 if (periph != NULL) {
 639                         struct cam_ed *device;
 640                         int i;
 641
 642                         base_periph_found = 1;
 643                         device = periph->path->device;
 644                         for (i = 0, periph = SLIST_FIRST(&device->periphs);
 645                              periph != NULL;
 646                              periph = SLIST_NEXT(periph, periph_links), i++) {
 647                                 /*
 648                                  * Check to see whether we have a
 649                                  * passthrough device or not.
 650                                  */
 651                                 if (strcmp(periph->periph_name, "pass") == 0) {
 652                                         /*
 653                                          * Fill in the getdevlist fields.
 654                                          */
 655                                         strcpy(ccb->cgdl.periph_name,
 656                                                periph->periph_name);
 657                                         ccb->cgdl.unit_number =
 658                                                 periph->unit_number;
 659                                         if (SLIST_NEXT(periph, periph_links))
 660                                                 ccb->cgdl.status =
 661                                                         CAM_GDEVLIST_MORE_DEVS;
 662                                         else
 663                                                 ccb->cgdl.status =
 664                                                        CAM_GDEVLIST_LAST_DEVICE;
 665                                         ccb->cgdl.generation =
 666                                                 device->generation;
 667                                         ccb->cgdl.index = i;
 668                                         /*
 669                                          * Fill in some CCB header fields
 670                                          * that the user may want.
 671                                          */
 672                                         ccb->ccb_h.path_id =
 673                                                 periph->path->bus->path_id;
 674                                         ccb->ccb_h.target_id =
 675                                                 periph->path->target->target_id;
 676                                         ccb->ccb_h.target_lun =
 677                                                 periph->path->device->lun_id;
 678                                         ccb->ccb_h.status = CAM_REQ_CMP;
 679                                         break;
 680                                 }
 681                         }
 682                 }
 683
 684                 /*
 685                  * If the periph is null here, one of two things has
 686                  * happened.  The first possibility is that we couldn't
 687                  * find the unit number of the particular peripheral driver
 688                  * that the user is asking about.  e.g. the user asks for
 689                  * the passthrough driver for "da11".  We find the list of
 690                  * "da" peripherals all right, but there is no unit 11.
 691                  * The other possibility is that we went through the list
 692                  * of peripheral drivers attached to the device structure,
 693                  * but didn't find one with the name "pass".  Either way,
 694                  * we return ENOENT, since we couldn't find something.
 695                  */
 696                 if (periph == NULL) {
 697                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 698                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 699                         *ccb->cgdl.periph_name = '\0';
 700                         ccb->cgdl.unit_number = 0;
 701                         error = ENOENT;
 702                         /*
 703                          * It is unfortunate that this is even necessary,
 704                          * but there are many, many clueless users out there.
 705                          * If this is true, the user is looking for the
 706                          * passthrough driver, but doesn't have one in his
 707                          * kernel.
 708                          */
 709                         if (base_periph_found == 1) {
 710                                 printf("xptioctl: pass driver is not in the "
 711                                        "kernel\n");
 712                                 printf("xptioctl: put \"device pass\" in "
 713                                        "your kernel config file\n");
 714                         }
 715                 }
 716                 xpt_unlock_buses();
 717                 break;
 718                 }
 719         default:
 720                 error = ENOTTY;
 721                 break;
 722         }
 723
 724         return(error);
 725 }
 726
 727 static int
 728 cam_module_event_handler(module_t mod, int what, void *arg)
 729 {
 730         int error;
 731
 732         switch (what) {
 733         case MOD_LOAD:
 734                 if ((error = xpt_init(NULL)) != 0)
 735                         return (error);
 736                 break;
 737         case MOD_UNLOAD:
 738                 return EBUSY;
 739         default:
 740                 return EOPNOTSUPP;
 741         }
 742
 743         return 0;
 744 }
 745
 746 static void
 747 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
 748 {
 749
 750         if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
 751                 xpt_free_path(done_ccb->ccb_h.path);
 752                 xpt_free_ccb(done_ccb);
 753         } else {
 754                 done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
 755                 (*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
 756         }
 757         xpt_release_boot();
 758 }
 759
 760 /* thread to handle bus rescans */
 761 static void
 762 xpt_scanner_thread(void *dummy)
 763 {
 764         union ccb       *ccb;
 765         struct cam_path  path;
 766
 767         xpt_lock_buses();
 768         for (;;) {
 769                 if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
 770                         msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
 771                                "-", 0);
 772                 if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
 773                         TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 774                         xpt_unlock_buses();
 775
 776                         /*
 777                          * Since lock can be dropped inside and path freed
 778                          * by completion callback even before return here,
 779                          * take our own path copy for reference.
 780                          */
 781                         xpt_copy_path(&path, ccb->ccb_h.path);
 782                         xpt_path_lock(&path);
 783                         xpt_action(ccb);
 784                         xpt_path_unlock(&path);
 785                         xpt_release_path(&path);
 786
 787                         xpt_lock_buses();
 788                 }
 789         }
 790 }
 791
 792 void
 793 xpt_rescan(union ccb *ccb)
 794 {
 795         struct ccb_hdr *hdr;
 796
 797         /* Prepare request */
 798         if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
 799             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 800                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 801         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 802             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 803                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 804         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 805             ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
 806                 ccb->ccb_h.func_code = XPT_SCAN_LUN;
 807         else {
 808                 xpt_print(ccb->ccb_h.path, "illegal scan path\n");
 809                 xpt_free_path(ccb->ccb_h.path);
 810                 xpt_free_ccb(ccb);
 811                 return;
 812         }
 813         ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
 814         ccb->ccb_h.cbfcnp = xpt_rescan_done;
 815         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
 816         /* Don't make duplicate entries for the same paths. */
 817         xpt_lock_buses();
 818         if (ccb->ccb_h.ppriv_ptr1 == NULL) {
 819                 TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
 820                         if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
 821                                 wakeup(&xsoftc.ccb_scanq);
 822                                 xpt_unlock_buses();
 823                                 xpt_print(ccb->ccb_h.path, "rescan already queued\n");
 824                                 xpt_free_path(ccb->ccb_h.path);
 825                                 xpt_free_ccb(ccb);
 826                                 return;
 827                         }
 828                 }
 829         }
 830         TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 831         xsoftc.buses_to_config++;
 832         wakeup(&xsoftc.ccb_scanq);
 833         xpt_unlock_buses();
 834 }
 835
 836 /* Functions accessed by the peripheral drivers */
 837 static int
 838 xpt_init(void *dummy)
 839 {
 840         struct cam_sim *xpt_sim;
 841         struct cam_path *path;
 842         struct cam_devq *devq;
 843         cam_status status;
 844         int error, i;
 845
 846         TAILQ_INIT(&xsoftc.xpt_busses);
 847         TAILQ_INIT(&xsoftc.ccb_scanq);
 848         STAILQ_INIT(&xsoftc.highpowerq);
 849         xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
 850
 851         mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
 852         mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
 853         mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF);
 854         xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
 855             taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
 856
 857 #ifdef CAM_BOOT_DELAY
 858         /*
 859          * Override this value at compile time to assist our users
 860          * who don't use loader to boot a kernel.
 861          */
 862         xsoftc.boot_delay = CAM_BOOT_DELAY;
 863 #endif
 864         /*
 865          * The xpt layer is, itself, the equivelent of a SIM.
 866          * Allow 16 ccbs in the ccb pool for it.  This should
 867          * give decent parallelism when we probe busses and
 868          * perform other XPT functions.
 869          */
 870         devq = cam_simq_alloc(16);
 871         xpt_sim = cam_sim_alloc(xptaction,
 872                                 xptpoll,
 873                                 "xpt",
 874                                 /*softc*/NULL,
 875                                 /*unit*/0,
 876                                 /*mtx*/&xsoftc.xpt_lock,
 877                                 /*max_dev_transactions*/0,
 878                                 /*max_tagged_dev_transactions*/0,
 879                                 devq);
 880         if (xpt_sim == NULL)
 881                 return (ENOMEM);
 882
 883         mtx_lock(&xsoftc.xpt_lock);
 884         if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
 885                 mtx_unlock(&xsoftc.xpt_lock);
 886                 printf("xpt_init: xpt_bus_register failed with status %#x,"
 887                        " failing attach\n", status);
 888                 return (EINVAL);
 889         }
 890         mtx_unlock(&xsoftc.xpt_lock);
 891
 892         /*
 893          * Looking at the XPT from the SIM layer, the XPT is
 894          * the equivelent of a peripheral driver.  Allocate
 895          * a peripheral driver entry for us.
 896          */
 897         if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
 898                                       CAM_TARGET_WILDCARD,
 899                                       CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
 900                 printf("xpt_init: xpt_create_path failed with status %#x,"
 901                        " failing attach\n", status);
 902                 return (EINVAL);
 903         }
 904         xpt_path_lock(path);
 905         cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
 906                          path, NULL, 0, xpt_sim);
 907         xpt_path_unlock(path);
 908         xpt_free_path(path);
 909
 910         if (cam_num_doneqs < 1)
 911                 cam_num_doneqs = 1 + mp_ncpus / 6;
 912         else if (cam_num_doneqs > MAXCPU)
 913                 cam_num_doneqs = MAXCPU;
 914         for (i = 0; i < cam_num_doneqs; i++) {
 915                 mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
 916                     MTX_DEF);
 917                 STAILQ_INIT(&cam_doneqs[i].cam_doneq);
 918                 error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
 919                     &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
 920                 if (error != 0) {
 921                         cam_num_doneqs = i;
 922                         break;
 923                 }
 924         }
 925         if (cam_num_doneqs < 1) {
 926                 printf("xpt_init: Cannot init completion queues "
 927                        "- failing attach\n");
 928                 return (ENOMEM);
 929         }
 930         /*
 931          * Register a callback for when interrupts are enabled.
 932          */
 933         xsoftc.xpt_config_hook =
 934             (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
 935                                               M_CAMXPT, M_NOWAIT | M_ZERO);
 936         if (xsoftc.xpt_config_hook == NULL) {
 937                 printf("xpt_init: Cannot malloc config hook "
 938                        "- failing attach\n");
 939                 return (ENOMEM);
 940         }
 941         xsoftc.xpt_config_hook->ich_func = xpt_config;
 942         if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
 943                 free (xsoftc.xpt_config_hook, M_CAMXPT);
 944                 printf("xpt_init: config_intrhook_establish failed "
 945                        "- failing attach\n");
 946         }
 947
 948         return (0);
 949 }
 950
 951 static cam_status
 952 xptregister(struct cam_periph *periph, void *arg)
 953 {
 954         struct cam_sim *xpt_sim;
 955
 956         if (periph == NULL) {
 957                 printf("xptregister: periph was NULL!!\n");
 958                 return(CAM_REQ_CMP_ERR);
 959         }
 960
 961         xpt_sim = (struct cam_sim *)arg;
 962         xpt_sim->softc = periph;
 963         xpt_periph = periph;
 964         periph->softc = NULL;
 965
 966         return(CAM_REQ_CMP);
 967 }
 968
 969 int32_t
 970 xpt_add_periph(struct cam_periph *periph)
 971 {
 972         struct cam_ed *device;
 973         int32_t  status;
 974
 975         TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
 976         device = periph->path->device;
 977         status = CAM_REQ_CMP;
 978         if (device != NULL) {
 979                 mtx_lock(&device->target->bus->eb_mtx);
 980                 device->generation++;
 981                 SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
 982                 mtx_unlock(&device->target->bus->eb_mtx);
 983         }
 984
 985         return (status);
 986 }
 987
 988 void
 989 xpt_remove_periph(struct cam_periph *periph)
 990 {
 991         struct cam_ed *device;
 992
 993         device = periph->path->device;
 994         if (device != NULL) {
 995                 mtx_lock(&device->target->bus->eb_mtx);
 996                 device->generation++;
 997                 SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
 998                 mtx_unlock(&device->target->bus->eb_mtx);
 999         }
1000 }
1001
1002
1003 void
1004 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1005 {
1006         struct  cam_path *path = periph->path;
1007
1008         cam_periph_assert(periph, MA_OWNED);
1009         periph->flags |= CAM_PERIPH_ANNOUNCED;
1010
1011         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1012                periph->periph_name, periph->unit_number,
1013                path->bus->sim->sim_name,
1014                path->bus->sim->unit_number,
1015                path->bus->sim->bus_id,
1016                path->bus->path_id,
1017                path->target->target_id,
1018                (uintmax_t)path->device->lun_id);
1019         printf("%s%d: ", periph->periph_name, periph->unit_number);
1020         if (path->device->protocol == PROTO_SCSI)
1021                 scsi_print_inquiry(&path->device->inq_data);
1022         else if (path->device->protocol == PROTO_ATA ||
1023             path->device->protocol == PROTO_SATAPM)
1024                 ata_print_ident(&path->device->ident_data);
1025         else if (path->device->protocol == PROTO_SEMB)
1026                 semb_print_ident(
1027                     (struct sep_identify_data *)&path->device->ident_data);
1028         else
1029                 printf("Unknown protocol device\n");
1030         if (path->device->serial_num_len > 0) {
1031                 /* Don't wrap the screen  - print only the first 60 chars */
1032                 printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1033                        periph->unit_number, path->device->serial_num);
1034         }
1035         /* Announce transport details. */
1036         (*(path->bus->xport->announce))(periph);
1037         /* Announce command queueing. */
1038         if (path->device->inq_flags & SID_CmdQue
1039          || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1040                 printf("%s%d: Command Queueing enabled\n",
1041                        periph->periph_name, periph->unit_number);
1042         }
1043         /* Announce caller's details if they've passed in. */
1044         if (announce_string != NULL)
1045                 printf("%s%d: %s\n", periph->periph_name,
1046                        periph->unit_number, announce_string);
1047 }
1048
1049 void
1050 xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1051 {
1052         if (quirks != 0) {
1053                 printf("%s%d: quirks=0x%b\n", periph->periph_name,
1054                     periph->unit_number, quirks, bit_string);
1055         }
1056 }
1057
1058 void
1059 xpt_denounce_periph(struct cam_periph *periph)
1060 {
1061         struct  cam_path *path = periph->path;
1062
1063         cam_periph_assert(periph, MA_OWNED);
1064         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1065                periph->periph_name, periph->unit_number,
1066                path->bus->sim->sim_name,
1067                path->bus->sim->unit_number,
1068                path->bus->sim->bus_id,
1069                path->bus->path_id,
1070                path->target->target_id,
1071                (uintmax_t)path->device->lun_id);
1072         printf("%s%d: ", periph->periph_name, periph->unit_number);
1073         if (path->device->protocol == PROTO_SCSI)
1074                 scsi_print_inquiry_short(&path->device->inq_data);
1075         else if (path->device->protocol == PROTO_ATA ||
1076             path->device->protocol == PROTO_SATAPM)
1077                 ata_print_ident_short(&path->device->ident_data);
1078         else if (path->device->protocol == PROTO_SEMB)
1079                 semb_print_ident_short(
1080                     (struct sep_identify_data *)&path->device->ident_data);
1081         else
1082                 printf("Unknown protocol device");
1083         if (path->device->serial_num_len > 0)
1084                 printf(" s/n %.60s", path->device->serial_num);
1085         printf(" detached\n");
1086 }
1087
1088
1089 int
1090 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1091 {
1092         int ret = -1, l;
1093         struct ccb_dev_advinfo cdai;
1094         struct scsi_vpd_id_descriptor *idd;
1095
1096         xpt_path_assert(path, MA_OWNED);
1097
1098         memset(&cdai, 0, sizeof(cdai));
1099         xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1100         cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1101         cdai.bufsiz = len;
1102
1103         if (!strcmp(attr, "GEOM::ident"))
1104                 cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1105         else if (!strcmp(attr, "GEOM::physpath"))
1106                 cdai.buftype = CDAI_TYPE_PHYS_PATH;
1107         else if (strcmp(attr, "GEOM::lunid") == 0 ||
1108                  strcmp(attr, "GEOM::lunname") == 0) {
1109                 cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1110                 cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1111         } else
1112                 goto out;
1113
1114         cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1115         if (cdai.buf == NULL) {
1116                 ret = ENOMEM;
1117                 goto out;
1118         }
1119         xpt_action((union ccb *)&cdai); /* can only be synchronous */
1120         if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1121                 cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1122         if (cdai.provsiz == 0)
1123                 goto out;
1124         if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1125                 if (strcmp(attr, "GEOM::lunid") == 0) {
1126                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1127                             cdai.provsiz, scsi_devid_is_lun_naa);
1128                         if (idd == NULL)
1129                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1130                                     cdai.provsiz, scsi_devid_is_lun_eui64);
1131                 } else
1132                         idd = NULL;
1133                 if (idd == NULL)
1134                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1135                             cdai.provsiz, scsi_devid_is_lun_t10);
1136                 if (idd == NULL)
1137                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1138                             cdai.provsiz, scsi_devid_is_lun_name);
1139                 if (idd == NULL)
1140                         goto out;
1141                 ret = 0;
1142                 if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1143                         if (idd->length < len) {
1144                                 for (l = 0; l < idd->length; l++)
1145                                         buf[l] = idd->identifier[l] ?
1146                                             idd->identifier[l] : ' ';
1147                                 buf[l] = 0;
1148                         } else
1149                                 ret = EFAULT;
1150                 } else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1151                         l = strnlen(idd->identifier, idd->length);
1152                         if (l < len) {
1153                                 bcopy(idd->identifier, buf, l);
1154                                 buf[l] = 0;
1155                         } else
1156                                 ret = EFAULT;
1157                 } else {
1158                         if (idd->length * 2 < len) {
1159                                 for (l = 0; l < idd->length; l++)
1160                                         sprintf(buf + l * 2, "%02x",
1161                                             idd->identifier[l]);
1162                         } else
1163                                 ret = EFAULT;
1164                 }
1165         } else {
1166                 ret = 0;
1167                 if (strlcpy(buf, cdai.buf, len) >= len)
1168                         ret = EFAULT;
1169         }
1170
1171 out:
1172         if (cdai.buf != NULL)
1173                 free(cdai.buf, M_CAMXPT);
1174         return ret;
1175 }
1176
1177 static dev_match_ret
1178 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1179             struct cam_eb *bus)
1180 {
1181         dev_match_ret retval;
1182         int i;
1183
1184         retval = DM_RET_NONE;
1185
1186         /*
1187          * If we aren't given something to match against, that's an error.
1188          */
1189         if (bus == NULL)
1190                 return(DM_RET_ERROR);
1191
1192         /*
1193          * If there are no match entries, then this bus matches no
1194          * matter what.
1195          */
1196         if ((patterns == NULL) || (num_patterns == 0))
1197                 return(DM_RET_DESCEND | DM_RET_COPY);
1198
1199         for (i = 0; i < num_patterns; i++) {
1200                 struct bus_match_pattern *cur_pattern;
1201
1202                 /*
1203                  * If the pattern in question isn't for a bus node, we
1204                  * aren't interested.  However, we do indicate to the
1205                  * calling routine that we should continue descending the
1206                  * tree, since the user wants to match against lower-level
1207                  * EDT elements.
1208                  */
1209                 if (patterns[i].type != DEV_MATCH_BUS) {
1210                         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1211                                 retval |= DM_RET_DESCEND;
1212                         continue;
1213                 }
1214
1215                 cur_pattern = &patterns[i].pattern.bus_pattern;
1216
1217                 /*
1218                  * If they want to match any bus node, we give them any
1219                  * device node.
1220                  */
1221                 if (cur_pattern->flags == BUS_MATCH_ANY) {
1222                         /* set the copy flag */
1223                         retval |= DM_RET_COPY;
1224
1225                         /*
1226                          * If we've already decided on an action, go ahead
1227                          * and return.
1228                          */
1229                         if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1230                                 return(retval);
1231                 }
1232
1233                 /*
1234                  * Not sure why someone would do this...
1235                  */
1236                 if (cur_pattern->flags == BUS_MATCH_NONE)
1237                         continue;
1238
1239                 if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1240                  && (cur_pattern->path_id != bus->path_id))
1241                         continue;
1242
1243                 if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1244                  && (cur_pattern->bus_id != bus->sim->bus_id))
1245                         continue;
1246
1247                 if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1248                  && (cur_pattern->unit_number != bus->sim->unit_number))
1249                         continue;
1250
1251                 if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1252                  && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1253                              DEV_IDLEN) != 0))
1254                         continue;
1255
1256                 /*
1257                  * If we get to this point, the user definitely wants
1258                  * information on this bus.  So tell the caller to copy the
1259                  * data out.
1260                  */
1261                 retval |= DM_RET_COPY;
1262
1263                 /*
1264                  * If the return action has been set to descend, then we
1265                  * know that we've already seen a non-bus matching
1266                  * expression, therefore we need to further descend the tree.
1267                  * This won't change by continuing around the loop, so we
1268                  * go ahead and return.  If we haven't seen a non-bus
1269                  * matching expression, we keep going around the loop until
1270                  * we exhaust the matching expressions.  We'll set the stop
1271                  * flag once we fall out of the loop.
1272                  */
1273                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1274                         return(retval);
1275         }
1276
1277         /*
1278          * If the return action hasn't been set to descend yet, that means
1279          * we haven't seen anything other than bus matching patterns.  So
1280          * tell the caller to stop descending the tree -- the user doesn't
1281          * want to match against lower level tree elements.
1282          */
1283         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1284                 retval |= DM_RET_STOP;
1285
1286         return(retval);
1287 }
1288
1289 static dev_match_ret
1290 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1291                struct cam_ed *device)
1292 {
1293         dev_match_ret retval;
1294         int i;
1295
1296         retval = DM_RET_NONE;
1297
1298         /*
1299          * If we aren't given something to match against, that's an error.
1300          */
1301         if (device == NULL)
1302                 return(DM_RET_ERROR);
1303
1304         /*
1305          * If there are no match entries, then this device matches no
1306          * matter what.
1307          */
1308         if ((patterns == NULL) || (num_patterns == 0))
1309                 return(DM_RET_DESCEND | DM_RET_COPY);
1310
1311         for (i = 0; i < num_patterns; i++) {
1312                 struct device_match_pattern *cur_pattern;
1313                 struct scsi_vpd_device_id *device_id_page;
1314
1315                 /*
1316                  * If the pattern in question isn't for a device node, we
1317                  * aren't interested.
1318                  */
1319                 if (patterns[i].type != DEV_MATCH_DEVICE) {
1320                         if ((patterns[i].type == DEV_MATCH_PERIPH)
1321                          && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1322                                 retval |= DM_RET_DESCEND;
1323                         continue;
1324                 }
1325
1326                 cur_pattern = &patterns[i].pattern.device_pattern;
1327
1328                 /* Error out if mutually exclusive options are specified. */
1329                 if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1330                  == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1331                         return(DM_RET_ERROR);
1332
1333                 /*
1334                  * If they want to match any device node, we give them any
1335                  * device node.
1336                  */
1337                 if (cur_pattern->flags == DEV_MATCH_ANY)
1338                         goto copy_dev_node;
1339
1340                 /*
1341                  * Not sure why someone would do this...
1342                  */
1343                 if (cur_pattern->flags == DEV_MATCH_NONE)
1344                         continue;
1345
1346                 if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1347                  && (cur_pattern->path_id != device->target->bus->path_id))
1348                         continue;
1349
1350                 if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1351                  && (cur_pattern->target_id != device->target->target_id))
1352                         continue;
1353
1354                 if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1355                  && (cur_pattern->target_lun != device->lun_id))
1356                         continue;
1357
1358                 if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1359                  && (cam_quirkmatch((caddr_t)&device->inq_data,
1360                                     (caddr_t)&cur_pattern->data.inq_pat,
1361                                     1, sizeof(cur_pattern->data.inq_pat),
1362                                     scsi_static_inquiry_match) == NULL))
1363                         continue;
1364
1365                 device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1366                 if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1367                  && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1368                   || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1369                                       device->device_id_len
1370                                     - SVPD_DEVICE_ID_HDR_LEN,
1371                                       cur_pattern->data.devid_pat.id,
1372                                       cur_pattern->data.devid_pat.id_len) != 0))
1373                         continue;
1374
1375 copy_dev_node:
1376                 /*
1377                  * If we get to this point, the user definitely wants
1378                  * information on this device.  So tell the caller to copy
1379                  * the data out.
1380                  */
1381                 retval |= DM_RET_COPY;
1382
1383                 /*
1384                  * If the return action has been set to descend, then we
1385                  * know that we've already seen a peripheral matching
1386                  * expression, therefore we need to further descend the tree.
1387                  * This won't change by continuing around the loop, so we
1388                  * go ahead and return.  If we haven't seen a peripheral
1389                  * matching expression, we keep going around the loop until
1390                  * we exhaust the matching expressions.  We'll set the stop
1391                  * flag once we fall out of the loop.
1392                  */
1393                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1394                         return(retval);
1395         }
1396
1397         /*
1398          * If the return action hasn't been set to descend yet, that means
1399          * we haven't seen any peripheral matching patterns.  So tell the
1400          * caller to stop descending the tree -- the user doesn't want to
1401          * match against lower level tree elements.
1402          */
1403         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1404                 retval |= DM_RET_STOP;
1405
1406         return(retval);
1407 }
1408
1409 /*
1410  * Match a single peripheral against any number of match patterns.
1411  */
1412 static dev_match_ret
1413 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1414                struct cam_periph *periph)
1415 {
1416         dev_match_ret retval;
1417         int i;
1418
1419         /*
1420          * If we aren't given something to match against, that's an error.
1421          */
1422         if (periph == NULL)
1423                 return(DM_RET_ERROR);
1424
1425         /*
1426          * If there are no match entries, then this peripheral matches no
1427          * matter what.
1428          */
1429         if ((patterns == NULL) || (num_patterns == 0))
1430                 return(DM_RET_STOP | DM_RET_COPY);
1431
1432         /*
1433          * There aren't any nodes below a peripheral node, so there's no
1434          * reason to descend the tree any further.
1435          */
1436         retval = DM_RET_STOP;
1437
1438         for (i = 0; i < num_patterns; i++) {
1439                 struct periph_match_pattern *cur_pattern;
1440
1441                 /*
1442                  * If the pattern in question isn't for a peripheral, we
1443                  * aren't interested.
1444                  */
1445                 if (patterns[i].type != DEV_MATCH_PERIPH)
1446                         continue;
1447
1448                 cur_pattern = &patterns[i].pattern.periph_pattern;
1449
1450                 /*
1451                  * If they want to match on anything, then we will do so.
1452                  */
1453                 if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1454                         /* set the copy flag */
1455                         retval |= DM_RET_COPY;
1456
1457                         /*
1458                          * We've already set the return action to stop,
1459                          * since there are no nodes below peripherals in
1460                          * the tree.
1461                          */
1462                         return(retval);
1463                 }
1464
1465                 /*
1466                  * Not sure why someone would do this...
1467                  */
1468                 if (cur_pattern->flags == PERIPH_MATCH_NONE)
1469                         continue;
1470
1471                 if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1472                  && (cur_pattern->path_id != periph->path->bus->path_id))
1473                         continue;
1474
1475                 /*
1476                  * For the target and lun id's, we have to make sure the
1477                  * target and lun pointers aren't NULL.  The xpt peripheral
1478                  * has a wildcard target and device.
1479                  */
1480                 if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1481                  && ((periph->path->target == NULL)
1482                  ||(cur_pattern->target_id != periph->path->target->target_id)))
1483                         continue;
1484
1485                 if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1486                  && ((periph->path->device == NULL)
1487                  || (cur_pattern->target_lun != periph->path->device->lun_id)))
1488                         continue;
1489
1490                 if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1491                  && (cur_pattern->unit_number != periph->unit_number))
1492                         continue;
1493
1494                 if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1495                  && (strncmp(cur_pattern->periph_name, periph->periph_name,
1496                              DEV_IDLEN) != 0))
1497                         continue;
1498
1499                 /*
1500                  * If we get to this point, the user definitely wants
1501                  * information on this peripheral.  So tell the caller to
1502                  * copy the data out.
1503                  */
1504                 retval |= DM_RET_COPY;
1505
1506                 /*
1507                  * The return action has already been set to stop, since
1508                  * peripherals don't have any nodes below them in the EDT.
1509                  */
1510                 return(retval);
1511         }
1512
1513         /*
1514          * If we get to this point, the peripheral that was passed in
1515          * doesn't match any of the patterns.
1516          */
1517         return(retval);
1518 }
1519
1520 static int
1521 xptedtbusfunc(struct cam_eb *bus, void *arg)
1522 {
1523         struct ccb_dev_match *cdm;
1524         struct cam_et *target;
1525         dev_match_ret retval;
1526
1527         cdm = (struct ccb_dev_match *)arg;
1528
1529         /*
1530          * If our position is for something deeper in the tree, that means
1531          * that we've already seen this node.  So, we keep going down.
1532          */
1533         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1534          && (cdm->pos.cookie.bus == bus)
1535          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1536          && (cdm->pos.cookie.target != NULL))
1537                 retval = DM_RET_DESCEND;
1538         else
1539                 retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1540
1541         /*
1542          * If we got an error, bail out of the search.
1543          */
1544         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1545                 cdm->status = CAM_DEV_MATCH_ERROR;
1546                 return(0);
1547         }
1548
1549         /*
1550          * If the copy flag is set, copy this bus out.
1551          */
1552         if (retval & DM_RET_COPY) {
1553                 int spaceleft, j;
1554
1555                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1556                         sizeof(struct dev_match_result));
1557
1558                 /*
1559                  * If we don't have enough space to put in another
1560                  * match result, save our position and tell the
1561                  * user there are more devices to check.
1562                  */
1563                 if (spaceleft < sizeof(struct dev_match_result)) {
1564                         bzero(&cdm->pos, sizeof(cdm->pos));
1565                         cdm->pos.position_type =
1566                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1567
1568                         cdm->pos.cookie.bus = bus;
1569                         cdm->pos.generations[CAM_BUS_GENERATION]=
1570                                 xsoftc.bus_generation;
1571                         cdm->status = CAM_DEV_MATCH_MORE;
1572                         return(0);
1573                 }
1574                 j = cdm->num_matches;
1575                 cdm->num_matches++;
1576                 cdm->matches[j].type = DEV_MATCH_BUS;
1577                 cdm->matches[j].result.bus_result.path_id = bus->path_id;
1578                 cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1579                 cdm->matches[j].result.bus_result.unit_number =
1580                         bus->sim->unit_number;
1581                 strncpy(cdm->matches[j].result.bus_result.dev_name,
1582                         bus->sim->sim_name, DEV_IDLEN);
1583         }
1584
1585         /*
1586          * If the user is only interested in busses, there's no
1587          * reason to descend to the next level in the tree.
1588          */
1589         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1590                 return(1);
1591
1592         /*
1593          * If there is a target generation recorded, check it to
1594          * make sure the target list hasn't changed.
1595          */
1596         mtx_lock(&bus->eb_mtx);
1597         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1598          && (cdm->pos.cookie.bus == bus)
1599          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1600          && (cdm->pos.cookie.target != NULL)) {
1601                 if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1602                     bus->generation)) {
1603                         mtx_unlock(&bus->eb_mtx);
1604                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1605                         return (0);
1606                 }
1607                 target = (struct cam_et *)cdm->pos.cookie.target;
1608                 target->refcount++;
1609         } else
1610                 target = NULL;
1611         mtx_unlock(&bus->eb_mtx);
1612
1613         return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1614 }
1615
1616 static int
1617 xptedttargetfunc(struct cam_et *target, void *arg)
1618 {
1619         struct ccb_dev_match *cdm;
1620         struct cam_eb *bus;
1621         struct cam_ed *device;
1622
1623         cdm = (struct ccb_dev_match *)arg;
1624         bus = target->bus;
1625
1626         /*
1627          * If there is a device list generation recorded, check it to
1628          * make sure the device list hasn't changed.
1629          */
1630         mtx_lock(&bus->eb_mtx);
1631         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1632          && (cdm->pos.cookie.bus == bus)
1633          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1634          && (cdm->pos.cookie.target == target)
1635          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1636          && (cdm->pos.cookie.device != NULL)) {
1637                 if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1638                     target->generation) {
1639                         mtx_unlock(&bus->eb_mtx);
1640                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1641                         return(0);
1642                 }
1643                 device = (struct cam_ed *)cdm->pos.cookie.device;
1644                 device->refcount++;
1645         } else
1646                 device = NULL;
1647         mtx_unlock(&bus->eb_mtx);
1648
1649         return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1650 }
1651
1652 static int
1653 xptedtdevicefunc(struct cam_ed *device, void *arg)
1654 {
1655         struct cam_eb *bus;
1656         struct cam_periph *periph;
1657         struct ccb_dev_match *cdm;
1658         dev_match_ret retval;
1659
1660         cdm = (struct ccb_dev_match *)arg;
1661         bus = device->target->bus;
1662
1663         /*
1664          * If our position is for something deeper in the tree, that means
1665          * that we've already seen this node.  So, we keep going down.
1666          */
1667         if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1668          && (cdm->pos.cookie.device == device)
1669          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1670          && (cdm->pos.cookie.periph != NULL))
1671                 retval = DM_RET_DESCEND;
1672         else
1673                 retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1674                                         device);
1675
1676         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1677                 cdm->status = CAM_DEV_MATCH_ERROR;
1678                 return(0);
1679         }
1680
1681         /*
1682          * If the copy flag is set, copy this device out.
1683          */
1684         if (retval & DM_RET_COPY) {
1685                 int spaceleft, j;
1686
1687                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1688                         sizeof(struct dev_match_result));
1689
1690                 /*
1691                  * If we don't have enough space to put in another
1692                  * match result, save our position and tell the
1693                  * user there are more devices to check.
1694                  */
1695                 if (spaceleft < sizeof(struct dev_match_result)) {
1696                         bzero(&cdm->pos, sizeof(cdm->pos));
1697                         cdm->pos.position_type =
1698                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1699                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1700
1701                         cdm->pos.cookie.bus = device->target->bus;
1702                         cdm->pos.generations[CAM_BUS_GENERATION]=
1703                                 xsoftc.bus_generation;
1704                         cdm->pos.cookie.target = device->target;
1705                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1706                                 device->target->bus->generation;
1707                         cdm->pos.cookie.device = device;
1708                         cdm->pos.generations[CAM_DEV_GENERATION] =
1709                                 device->target->generation;
1710                         cdm->status = CAM_DEV_MATCH_MORE;
1711                         return(0);
1712                 }
1713                 j = cdm->num_matches;
1714                 cdm->num_matches++;
1715                 cdm->matches[j].type = DEV_MATCH_DEVICE;
1716                 cdm->matches[j].result.device_result.path_id =
1717                         device->target->bus->path_id;
1718                 cdm->matches[j].result.device_result.target_id =
1719                         device->target->target_id;
1720                 cdm->matches[j].result.device_result.target_lun =
1721                         device->lun_id;
1722                 cdm->matches[j].result.device_result.protocol =
1723                         device->protocol;
1724                 bcopy(&device->inq_data,
1725                       &cdm->matches[j].result.device_result.inq_data,
1726                       sizeof(struct scsi_inquiry_data));
1727                 bcopy(&device->ident_data,
1728                       &cdm->matches[j].result.device_result.ident_data,
1729                       sizeof(struct ata_params));
1730
1731                 /* Let the user know whether this device is unconfigured */
1732                 if (device->flags & CAM_DEV_UNCONFIGURED)
1733                         cdm->matches[j].result.device_result.flags =
1734                                 DEV_RESULT_UNCONFIGURED;
1735                 else
1736                         cdm->matches[j].result.device_result.flags =
1737                                 DEV_RESULT_NOFLAG;
1738         }
1739
1740         /*
1741          * If the user isn't interested in peripherals, don't descend
1742          * the tree any further.
1743          */
1744         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1745                 return(1);
1746
1747         /*
1748          * If there is a peripheral list generation recorded, make sure
1749          * it hasn't changed.
1750          */
1751         xpt_lock_buses();
1752         mtx_lock(&bus->eb_mtx);
1753         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1754          && (cdm->pos.cookie.bus == bus)
1755          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1756          && (cdm->pos.cookie.target == device->target)
1757          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1758          && (cdm->pos.cookie.device == device)
1759          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1760          && (cdm->pos.cookie.periph != NULL)) {
1761                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1762                     device->generation) {
1763                         mtx_unlock(&bus->eb_mtx);
1764                         xpt_unlock_buses();
1765                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1766                         return(0);
1767                 }
1768                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1769                 periph->refcount++;
1770         } else
1771                 periph = NULL;
1772         mtx_unlock(&bus->eb_mtx);
1773         xpt_unlock_buses();
1774
1775         return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1776 }
1777
1778 static int
1779 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1780 {
1781         struct ccb_dev_match *cdm;
1782         dev_match_ret retval;
1783
1784         cdm = (struct ccb_dev_match *)arg;
1785
1786         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1787
1788         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1789                 cdm->status = CAM_DEV_MATCH_ERROR;
1790                 return(0);
1791         }
1792
1793         /*
1794          * If the copy flag is set, copy this peripheral out.
1795          */
1796         if (retval & DM_RET_COPY) {
1797                 int spaceleft, j;
1798
1799                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1800                         sizeof(struct dev_match_result));
1801
1802                 /*
1803                  * If we don't have enough space to put in another
1804                  * match result, save our position and tell the
1805                  * user there are more devices to check.
1806                  */
1807                 if (spaceleft < sizeof(struct dev_match_result)) {
1808                         bzero(&cdm->pos, sizeof(cdm->pos));
1809                         cdm->pos.position_type =
1810                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1811                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1812                                 CAM_DEV_POS_PERIPH;
1813
1814                         cdm->pos.cookie.bus = periph->path->bus;
1815                         cdm->pos.generations[CAM_BUS_GENERATION]=
1816                                 xsoftc.bus_generation;
1817                         cdm->pos.cookie.target = periph->path->target;
1818                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1819                                 periph->path->bus->generation;
1820                         cdm->pos.cookie.device = periph->path->device;
1821                         cdm->pos.generations[CAM_DEV_GENERATION] =
1822                                 periph->path->target->generation;
1823                         cdm->pos.cookie.periph = periph;
1824                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
1825                                 periph->path->device->generation;
1826                         cdm->status = CAM_DEV_MATCH_MORE;
1827                         return(0);
1828                 }
1829
1830                 j = cdm->num_matches;
1831                 cdm->num_matches++;
1832                 cdm->matches[j].type = DEV_MATCH_PERIPH;
1833                 cdm->matches[j].result.periph_result.path_id =
1834                         periph->path->bus->path_id;
1835                 cdm->matches[j].result.periph_result.target_id =
1836                         periph->path->target->target_id;
1837                 cdm->matches[j].result.periph_result.target_lun =
1838                         periph->path->device->lun_id;
1839                 cdm->matches[j].result.periph_result.unit_number =
1840                         periph->unit_number;
1841                 strncpy(cdm->matches[j].result.periph_result.periph_name,
1842                         periph->periph_name, DEV_IDLEN);
1843         }
1844
1845         return(1);
1846 }
1847
1848 static int
1849 xptedtmatch(struct ccb_dev_match *cdm)
1850 {
1851         struct cam_eb *bus;
1852         int ret;
1853
1854         cdm->num_matches = 0;
1855
1856         /*
1857          * Check the bus list generation.  If it has changed, the user
1858          * needs to reset everything and start over.
1859          */
1860         xpt_lock_buses();
1861         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1862          && (cdm->pos.cookie.bus != NULL)) {
1863                 if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1864                     xsoftc.bus_generation) {
1865                         xpt_unlock_buses();
1866                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1867                         return(0);
1868                 }
1869                 bus = (struct cam_eb *)cdm->pos.cookie.bus;
1870                 bus->refcount++;
1871         } else
1872                 bus = NULL;
1873         xpt_unlock_buses();
1874
1875         ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1876
1877         /*
1878          * If we get back 0, that means that we had to stop before fully
1879          * traversing the EDT.  It also means that one of the subroutines
1880          * has set the status field to the proper value.  If we get back 1,
1881          * we've fully traversed the EDT and copied out any matching entries.
1882          */
1883         if (ret == 1)
1884                 cdm->status = CAM_DEV_MATCH_LAST;
1885
1886         return(ret);
1887 }
1888
1889 static int
1890 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1891 {
1892         struct cam_periph *periph;
1893         struct ccb_dev_match *cdm;
1894
1895         cdm = (struct ccb_dev_match *)arg;
1896
1897         xpt_lock_buses();
1898         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1899          && (cdm->pos.cookie.pdrv == pdrv)
1900          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1901          && (cdm->pos.cookie.periph != NULL)) {
1902                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1903                     (*pdrv)->generation) {
1904                         xpt_unlock_buses();
1905                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1906                         return(0);
1907                 }
1908                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1909                 periph->refcount++;
1910         } else
1911                 periph = NULL;
1912         xpt_unlock_buses();
1913
1914         return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1915 }
1916
1917 static int
1918 xptplistperiphfunc(struct cam_periph *periph, void *arg)
1919 {
1920         struct ccb_dev_match *cdm;
1921         dev_match_ret retval;
1922
1923         cdm = (struct ccb_dev_match *)arg;
1924
1925         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1926
1927         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1928                 cdm->status = CAM_DEV_MATCH_ERROR;
1929                 return(0);
1930         }
1931
1932         /*
1933          * If the copy flag is set, copy this peripheral out.
1934          */
1935         if (retval & DM_RET_COPY) {
1936                 int spaceleft, j;
1937
1938                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1939                         sizeof(struct dev_match_result));
1940
1941                 /*
1942                  * If we don't have enough space to put in another
1943                  * match result, save our position and tell the
1944                  * user there are more devices to check.
1945                  */
1946                 if (spaceleft < sizeof(struct dev_match_result)) {
1947                         struct periph_driver **pdrv;
1948
1949                         pdrv = NULL;
1950                         bzero(&cdm->pos, sizeof(cdm->pos));
1951                         cdm->pos.position_type =
1952                                 CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1953                                 CAM_DEV_POS_PERIPH;
1954
1955                         /*
1956                          * This may look a bit non-sensical, but it is
1957                          * actually quite logical.  There are very few
1958                          * peripheral drivers, and bloating every peripheral
1959                          * structure with a pointer back to its parent
1960                          * peripheral driver linker set entry would cost
1961                          * more in the long run than doing this quick lookup.
1962                          */
1963                         for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1964                                 if (strcmp((*pdrv)->driver_name,
1965                                     periph->periph_name) == 0)
1966                                         break;
1967                         }
1968
1969                         if (*pdrv == NULL) {
1970                                 cdm->status = CAM_DEV_MATCH_ERROR;
1971                                 return(0);
1972                         }
1973
1974                         cdm->pos.cookie.pdrv = pdrv;
1975                         /*
1976                          * The periph generation slot does double duty, as
1977                          * does the periph pointer slot.  They are used for
1978                          * both edt and pdrv lookups and positioning.
1979                          */
1980                         cdm->pos.cookie.periph = periph;
1981                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
1982                                 (*pdrv)->generation;
1983                         cdm->status = CAM_DEV_MATCH_MORE;
1984                         return(0);
1985                 }
1986
1987                 j = cdm->num_matches;
1988                 cdm->num_matches++;
1989                 cdm->matches[j].type = DEV_MATCH_PERIPH;
1990                 cdm->matches[j].result.periph_result.path_id =
1991                         periph->path->bus->path_id;
1992
1993                 /*
1994                  * The transport layer peripheral doesn't have a target or
1995                  * lun.
1996                  */
1997                 if (periph->path->target)
1998                         cdm->matches[j].result.periph_result.target_id =
1999                                 periph->path->target->target_id;
2000                 else
2001                         cdm->matches[j].result.periph_result.target_id =
2002                                 CAM_TARGET_WILDCARD;
2003
2004                 if (periph->path->device)
2005                         cdm->matches[j].result.periph_result.target_lun =
2006                                 periph->path->device->lun_id;
2007                 else
2008                         cdm->matches[j].result.periph_result.target_lun =
2009                                 CAM_LUN_WILDCARD;
2010
2011                 cdm->matches[j].result.periph_result.unit_number =
2012                         periph->unit_number;
2013                 strncpy(cdm->matches[j].result.periph_result.periph_name,
2014                         periph->periph_name, DEV_IDLEN);
2015         }
2016
2017         return(1);
2018 }
2019
2020 static int
2021 xptperiphlistmatch(struct ccb_dev_match *cdm)
2022 {
2023         int ret;
2024
2025         cdm->num_matches = 0;
2026
2027         /*
2028          * At this point in the edt traversal function, we check the bus
2029          * list generation to make sure that no busses have been added or
2030          * removed since the user last sent a XPT_DEV_MATCH ccb through.
2031          * For the peripheral driver list traversal function, however, we
2032          * don't have to worry about new peripheral driver types coming or
2033          * going; they're in a linker set, and therefore can't change
2034          * without a recompile.
2035          */
2036
2037         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2038          && (cdm->pos.cookie.pdrv != NULL))
2039                 ret = xptpdrvtraverse(
2040                                 (struct periph_driver **)cdm->pos.cookie.pdrv,
2041                                 xptplistpdrvfunc, cdm);
2042         else
2043                 ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2044
2045         /*
2046          * If we get back 0, that means that we had to stop before fully
2047          * traversing the peripheral driver tree.  It also means that one of
2048          * the subroutines has set the status field to the proper value.  If
2049          * we get back 1, we've fully traversed the EDT and copied out any
2050          * matching entries.
2051          */
2052         if (ret == 1)
2053                 cdm->status = CAM_DEV_MATCH_LAST;
2054
2055         return(ret);
2056 }
2057
2058 static int
2059 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2060 {
2061         struct cam_eb *bus, *next_bus;
2062         int retval;
2063
2064         retval = 1;
2065         if (start_bus)
2066                 bus = start_bus;
2067         else {
2068                 xpt_lock_buses();
2069                 bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2070                 if (bus == NULL) {
2071                         xpt_unlock_buses();
2072                         return (retval);
2073                 }
2074                 bus->refcount++;
2075                 xpt_unlock_buses();
2076         }
2077         for (; bus != NULL; bus = next_bus) {
2078                 retval = tr_func(bus, arg);
2079                 if (retval == 0) {
2080                         xpt_release_bus(bus);
2081                         break;
2082                 }
2083                 xpt_lock_buses();
2084                 next_bus = TAILQ_NEXT(bus, links);
2085                 if (next_bus)
2086                         next_bus->refcount++;
2087                 xpt_unlock_buses();
2088                 xpt_release_bus(bus);
2089         }
2090         return(retval);
2091 }
2092
2093 static int
2094 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2095                   xpt_targetfunc_t *tr_func, void *arg)
2096 {
2097         struct cam_et *target, *next_target;
2098         int retval;
2099
2100         retval = 1;
2101         if (start_target)
2102                 target = start_target;
2103         else {
2104                 mtx_lock(&bus->eb_mtx);
2105                 target = TAILQ_FIRST(&bus->et_entries);
2106                 if (target == NULL) {
2107                         mtx_unlock(&bus->eb_mtx);
2108                         return (retval);
2109                 }
2110                 target->refcount++;
2111                 mtx_unlock(&bus->eb_mtx);
2112         }
2113         for (; target != NULL; target = next_target) {
2114                 retval = tr_func(target, arg);
2115                 if (retval == 0) {
2116                         xpt_release_target(target);
2117                         break;
2118                 }
2119                 mtx_lock(&bus->eb_mtx);
2120                 next_target = TAILQ_NEXT(target, links);
2121                 if (next_target)
2122                         next_target->refcount++;
2123                 mtx_unlock(&bus->eb_mtx);
2124                 xpt_release_target(target);
2125         }
2126         return(retval);
2127 }
2128
2129 static int
2130 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2131                   xpt_devicefunc_t *tr_func, void *arg)
2132 {
2133         struct cam_eb *bus;
2134         struct cam_ed *device, *next_device;
2135         int retval;
2136
2137         retval = 1;
2138         bus = target->bus;
2139         if (start_device)
2140                 device = start_device;
2141         else {
2142                 mtx_lock(&bus->eb_mtx);
2143                 device = TAILQ_FIRST(&target->ed_entries);
2144                 if (device == NULL) {
2145                         mtx_unlock(&bus->eb_mtx);
2146                         return (retval);
2147                 }
2148                 device->refcount++;
2149                 mtx_unlock(&bus->eb_mtx);
2150         }
2151         for (; device != NULL; device = next_device) {
2152                 mtx_lock(&device->device_mtx);
2153                 retval = tr_func(device, arg);
2154                 mtx_unlock(&device->device_mtx);
2155                 if (retval == 0) {
2156                         xpt_release_device(device);
2157                         break;
2158                 }
2159                 mtx_lock(&bus->eb_mtx);
2160                 next_device = TAILQ_NEXT(device, links);
2161                 if (next_device)
2162                         next_device->refcount++;
2163                 mtx_unlock(&bus->eb_mtx);
2164                 xpt_release_device(device);
2165         }
2166         return(retval);
2167 }
2168
2169 static int
2170 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2171                   xpt_periphfunc_t *tr_func, void *arg)
2172 {
2173         struct cam_eb *bus;
2174         struct cam_periph *periph, *next_periph;
2175         int retval;
2176
2177         retval = 1;
2178
2179         bus = device->target->bus;
2180         if (start_periph)
2181                 periph = start_periph;
2182         else {
2183                 xpt_lock_buses();
2184                 mtx_lock(&bus->eb_mtx);
2185                 periph = SLIST_FIRST(&device->periphs);
2186                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2187                         periph = SLIST_NEXT(periph, periph_links);
2188                 if (periph == NULL) {
2189                         mtx_unlock(&bus->eb_mtx);
2190                         xpt_unlock_buses();
2191                         return (retval);
2192                 }
2193                 periph->refcount++;
2194                 mtx_unlock(&bus->eb_mtx);
2195                 xpt_unlock_buses();
2196         }
2197         for (; periph != NULL; periph = next_periph) {
2198                 retval = tr_func(periph, arg);
2199                 if (retval == 0) {
2200                         cam_periph_release_locked(periph);
2201                         break;
2202                 }
2203                 xpt_lock_buses();
2204                 mtx_lock(&bus->eb_mtx);
2205                 next_periph = SLIST_NEXT(periph, periph_links);
2206                 while (next_periph != NULL &&
2207                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2208                         next_periph = SLIST_NEXT(next_periph, periph_links);
2209                 if (next_periph)
2210                         next_periph->refcount++;
2211                 mtx_unlock(&bus->eb_mtx);
2212                 xpt_unlock_buses();
2213                 cam_periph_release_locked(periph);
2214         }
2215         return(retval);
2216 }
2217
2218 static int
2219 xptpdrvtraverse(struct periph_driver **start_pdrv,
2220                 xpt_pdrvfunc_t *tr_func, void *arg)
2221 {
2222         struct periph_driver **pdrv;
2223         int retval;
2224
2225         retval = 1;
2226
2227         /*
2228          * We don't traverse the peripheral driver list like we do the
2229          * other lists, because it is a linker set, and therefore cannot be
2230          * changed during runtime.  If the peripheral driver list is ever
2231          * re-done to be something other than a linker set (i.e. it can
2232          * change while the system is running), the list traversal should
2233          * be modified to work like the other traversal functions.
2234          */
2235         for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2236              *pdrv != NULL; pdrv++) {
2237                 retval = tr_func(pdrv, arg);
2238
2239                 if (retval == 0)
2240                         return(retval);
2241         }
2242
2243         return(retval);
2244 }
2245
2246 static int
2247 xptpdperiphtraverse(struct periph_driver **pdrv,
2248                     struct cam_periph *start_periph,
2249                     xpt_periphfunc_t *tr_func, void *arg)
2250 {
2251         struct cam_periph *periph, *next_periph;
2252         int retval;
2253
2254         retval = 1;
2255
2256         if (start_periph)
2257                 periph = start_periph;
2258         else {
2259                 xpt_lock_buses();
2260                 periph = TAILQ_FIRST(&(*pdrv)->units);
2261                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2262                         periph = TAILQ_NEXT(periph, unit_links);
2263                 if (periph == NULL) {
2264                         xpt_unlock_buses();
2265                         return (retval);
2266                 }
2267                 periph->refcount++;
2268                 xpt_unlock_buses();
2269         }
2270         for (; periph != NULL; periph = next_periph) {
2271                 cam_periph_lock(periph);
2272                 retval = tr_func(periph, arg);
2273                 cam_periph_unlock(periph);
2274                 if (retval == 0) {
2275                         cam_periph_release(periph);
2276                         break;
2277                 }
2278                 xpt_lock_buses();
2279                 next_periph = TAILQ_NEXT(periph, unit_links);
2280                 while (next_periph != NULL &&
2281                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2282                         next_periph = TAILQ_NEXT(next_periph, unit_links);
2283                 if (next_periph)
2284                         next_periph->refcount++;
2285                 xpt_unlock_buses();
2286                 cam_periph_release(periph);
2287         }
2288         return(retval);
2289 }
2290
2291 static int
2292 xptdefbusfunc(struct cam_eb *bus, void *arg)
2293 {
2294         struct xpt_traverse_config *tr_config;
2295
2296         tr_config = (struct xpt_traverse_config *)arg;
2297
2298         if (tr_config->depth == XPT_DEPTH_BUS) {
2299                 xpt_busfunc_t *tr_func;
2300
2301                 tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2302
2303                 return(tr_func(bus, tr_config->tr_arg));
2304         } else
2305                 return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2306 }
2307
2308 static int
2309 xptdeftargetfunc(struct cam_et *target, void *arg)
2310 {
2311         struct xpt_traverse_config *tr_config;
2312
2313         tr_config = (struct xpt_traverse_config *)arg;
2314
2315         if (tr_config->depth == XPT_DEPTH_TARGET) {
2316                 xpt_targetfunc_t *tr_func;
2317
2318                 tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2319
2320                 return(tr_func(target, tr_config->tr_arg));
2321         } else
2322                 return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2323 }
2324
2325 static int
2326 xptdefdevicefunc(struct cam_ed *device, void *arg)
2327 {
2328         struct xpt_traverse_config *tr_config;
2329
2330         tr_config = (struct xpt_traverse_config *)arg;
2331
2332         if (tr_config->depth == XPT_DEPTH_DEVICE) {
2333                 xpt_devicefunc_t *tr_func;
2334
2335                 tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2336
2337                 return(tr_func(device, tr_config->tr_arg));
2338         } else
2339                 return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2340 }
2341
2342 static int
2343 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2344 {
2345         struct xpt_traverse_config *tr_config;
2346         xpt_periphfunc_t *tr_func;
2347
2348         tr_config = (struct xpt_traverse_config *)arg;
2349
2350         tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2351
2352         /*
2353          * Unlike the other default functions, we don't check for depth
2354          * here.  The peripheral driver level is the last level in the EDT,
2355          * so if we're here, we should execute the function in question.
2356          */
2357         return(tr_func(periph, tr_config->tr_arg));
2358 }
2359
2360 /*
2361  * Execute the given function for every bus in the EDT.
2362  */
2363 static int
2364 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2365 {
2366         struct xpt_traverse_config tr_config;
2367
2368         tr_config.depth = XPT_DEPTH_BUS;
2369         tr_config.tr_func = tr_func;
2370         tr_config.tr_arg = arg;
2371
2372         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2373 }
2374
2375 /*
2376  * Execute the given function for every device in the EDT.
2377  */
2378 static int
2379 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2380 {
2381         struct xpt_traverse_config tr_config;
2382
2383         tr_config.depth = XPT_DEPTH_DEVICE;
2384         tr_config.tr_func = tr_func;
2385         tr_config.tr_arg = arg;
2386
2387         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2388 }
2389
2390 static int
2391 xptsetasyncfunc(struct cam_ed *device, void *arg)
2392 {
2393         struct cam_path path;
2394         struct ccb_getdev cgd;
2395         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2396
2397         /*
2398          * Don't report unconfigured devices (Wildcard devs,
2399          * devices only for target mode, device instances
2400          * that have been invalidated but are waiting for
2401          * their last reference count to be released).
2402          */
2403         if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2404                 return (1);
2405
2406         xpt_compile_path(&path,
2407                          NULL,
2408                          device->target->bus->path_id,
2409                          device->target->target_id,
2410                          device->lun_id);
2411         xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2412         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2413         xpt_action((union ccb *)&cgd);
2414         csa->callback(csa->callback_arg,
2415                             AC_FOUND_DEVICE,
2416                             &path, &cgd);
2417         xpt_release_path(&path);
2418
2419         return(1);
2420 }
2421
2422 static int
2423 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2424 {
2425         struct cam_path path;
2426         struct ccb_pathinq cpi;
2427         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2428
2429         xpt_compile_path(&path, /*periph*/NULL,
2430                          bus->path_id,
2431                          CAM_TARGET_WILDCARD,
2432                          CAM_LUN_WILDCARD);
2433         xpt_path_lock(&path);
2434         xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2435         cpi.ccb_h.func_code = XPT_PATH_INQ;
2436         xpt_action((union ccb *)&cpi);
2437         csa->callback(csa->callback_arg,
2438                             AC_PATH_REGISTERED,
2439                             &path, &cpi);
2440         xpt_path_unlock(&path);
2441         xpt_release_path(&path);
2442
2443         return(1);
2444 }
2445
2446 void
2447 xpt_action(union ccb *start_ccb)
2448 {
2449
2450         CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2451
2452         start_ccb->ccb_h.status = CAM_REQ_INPROG;
2453         (*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2454 }
2455
2456 void
2457 xpt_action_default(union ccb *start_ccb)
2458 {
2459         struct cam_path *path;
2460         struct cam_sim *sim;
2461         int lock;
2462
2463         path = start_ccb->ccb_h.path;
2464         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2465
2466         switch (start_ccb->ccb_h.func_code) {
2467         case XPT_SCSI_IO:
2468         {
2469                 struct cam_ed *device;
2470
2471                 /*
2472                  * For the sake of compatibility with SCSI-1
2473                  * devices that may not understand the identify
2474                  * message, we include lun information in the
2475                  * second byte of all commands.  SCSI-1 specifies
2476                  * that luns are a 3 bit value and reserves only 3
2477                  * bits for lun information in the CDB.  Later
2478                  * revisions of the SCSI spec allow for more than 8
2479                  * luns, but have deprecated lun information in the
2480                  * CDB.  So, if the lun won't fit, we must omit.
2481                  *
2482                  * Also be aware that during initial probing for devices,
2483                  * the inquiry information is unknown but initialized to 0.
2484                  * This means that this code will be exercised while probing
2485                  * devices with an ANSI revision greater than 2.
2486                  */
2487                 device = path->device;
2488                 if (device->protocol_version <= SCSI_REV_2
2489                  && start_ccb->ccb_h.target_lun < 8
2490                  && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2491
2492                         start_ccb->csio.cdb_io.cdb_bytes[1] |=
2493                             start_ccb->ccb_h.target_lun << 5;
2494                 }
2495                 start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2496         }
2497         /* FALLTHROUGH */
2498         case XPT_TARGET_IO:
2499         case XPT_CONT_TARGET_IO:
2500                 start_ccb->csio.sense_resid = 0;
2501                 start_ccb->csio.resid = 0;
2502                 /* FALLTHROUGH */
2503         case XPT_ATA_IO:
2504                 if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2505                         start_ccb->ataio.resid = 0;
2506                 /* FALLTHROUGH */
2507         case XPT_RESET_DEV:
2508         case XPT_ENG_EXEC:
2509         case XPT_SMP_IO:
2510         {
2511                 struct cam_devq *devq;
2512
2513                 devq = path->bus->sim->devq;
2514                 mtx_lock(&devq->send_mtx);
2515                 cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2516                 if (xpt_schedule_devq(devq, path->device) != 0)
2517                         xpt_run_devq(devq);
2518                 mtx_unlock(&devq->send_mtx);
2519                 break;
2520         }
2521         case XPT_CALC_GEOMETRY:
2522                 /* Filter out garbage */
2523                 if (start_ccb->ccg.block_size == 0
2524                  || start_ccb->ccg.volume_size == 0) {
2525                         start_ccb->ccg.cylinders = 0;
2526                         start_ccb->ccg.heads = 0;
2527                         start_ccb->ccg.secs_per_track = 0;
2528                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2529                         break;
2530                 }
2531 #if defined(PC98) || defined(__sparc64__)
2532                 /*
2533                  * In a PC-98 system, geometry translation depens on
2534                  * the "real" device geometry obtained from mode page 4.
2535                  * SCSI geometry translation is performed in the
2536                  * initialization routine of the SCSI BIOS and the result
2537                  * stored in host memory.  If the translation is available
2538                  * in host memory, use it.  If not, rely on the default
2539                  * translation the device driver performs.
2540                  * For sparc64, we may need adjust the geometry of large
2541                  * disks in order to fit the limitations of the 16-bit
2542                  * fields of the VTOC8 disk label.
2543                  */
2544                 if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2545                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2546                         break;
2547                 }
2548 #endif
2549                 goto call_sim;
2550         case XPT_ABORT:
2551         {
2552                 union ccb* abort_ccb;
2553
2554                 abort_ccb = start_ccb->cab.abort_ccb;
2555                 if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2556
2557                         if (abort_ccb->ccb_h.pinfo.index >= 0) {
2558                                 struct cam_ccbq *ccbq;
2559                                 struct cam_ed *device;
2560
2561                                 device = abort_ccb->ccb_h.path->device;
2562                                 ccbq = &device->ccbq;
2563                                 cam_ccbq_remove_ccb(ccbq, abort_ccb);
2564                                 abort_ccb->ccb_h.status =
2565                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2566                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2567                                 xpt_done(abort_ccb);
2568                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2569                                 break;
2570                         }
2571                         if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2572                          && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2573                                 /*
2574                                  * We've caught this ccb en route to
2575                                  * the SIM.  Flag it for abort and the
2576                                  * SIM will do so just before starting
2577                                  * real work on the CCB.
2578                                  */
2579                                 abort_ccb->ccb_h.status =
2580                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2581                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2582                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2583                                 break;
2584                         }
2585                 }
2586                 if (XPT_FC_IS_QUEUED(abort_ccb)
2587                  && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2588                         /*
2589                          * It's already completed but waiting
2590                          * for our SWI to get to it.
2591                          */
2592                         start_ccb->ccb_h.status = CAM_UA_ABORT;
2593                         break;
2594                 }
2595                 /*
2596                  * If we weren't able to take care of the abort request
2597                  * in the XPT, pass the request down to the SIM for processing.
2598                  */
2599         }
2600         /* FALLTHROUGH */
2601         case XPT_ACCEPT_TARGET_IO:
2602         case XPT_EN_LUN:
2603         case XPT_IMMED_NOTIFY:
2604         case XPT_NOTIFY_ACK:
2605         case XPT_RESET_BUS:
2606         case XPT_IMMEDIATE_NOTIFY:
2607         case XPT_NOTIFY_ACKNOWLEDGE:
2608         case XPT_GET_SIM_KNOB:
2609         case XPT_SET_SIM_KNOB:
2610         case XPT_GET_TRAN_SETTINGS:
2611         case XPT_SET_TRAN_SETTINGS:
2612         case XPT_PATH_INQ:
2613 call_sim:
2614                 sim = path->bus->sim;
2615                 lock = (mtx_owned(sim->mtx) == 0);
2616                 if (lock)
2617                         CAM_SIM_LOCK(sim);
2618                 (*(sim->sim_action))(sim, start_ccb);
2619                 if (lock)
2620                         CAM_SIM_UNLOCK(sim);
2621                 break;
2622         case XPT_PATH_STATS:
2623                 start_ccb->cpis.last_reset = path->bus->last_reset;
2624                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2625                 break;
2626         case XPT_GDEV_TYPE:
2627         {
2628                 struct cam_ed *dev;
2629
2630                 dev = path->device;
2631                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2632                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2633                 } else {
2634                         struct ccb_getdev *cgd;
2635
2636                         cgd = &start_ccb->cgd;
2637                         cgd->protocol = dev->protocol;
2638                         cgd->inq_data = dev->inq_data;
2639                         cgd->ident_data = dev->ident_data;
2640                         cgd->inq_flags = dev->inq_flags;
2641                         cgd->ccb_h.status = CAM_REQ_CMP;
2642                         cgd->serial_num_len = dev->serial_num_len;
2643                         if ((dev->serial_num_len > 0)
2644                          && (dev->serial_num != NULL))
2645                                 bcopy(dev->serial_num, cgd->serial_num,
2646                                       dev->serial_num_len);
2647                 }
2648                 break;
2649         }
2650         case XPT_GDEV_STATS:
2651         {
2652                 struct cam_ed *dev;
2653
2654                 dev = path->device;
2655                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2656                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2657                 } else {
2658                         struct ccb_getdevstats *cgds;
2659                         struct cam_eb *bus;
2660                         struct cam_et *tar;
2661                         struct cam_devq *devq;
2662
2663                         cgds = &start_ccb->cgds;
2664                         bus = path->bus;
2665                         tar = path->target;
2666                         devq = bus->sim->devq;
2667                         mtx_lock(&devq->send_mtx);
2668                         cgds->dev_openings = dev->ccbq.dev_openings;
2669                         cgds->dev_active = dev->ccbq.dev_active;
2670                         cgds->allocated = dev->ccbq.allocated;
2671                         cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2672                         cgds->held = cgds->allocated - cgds->dev_active -
2673                             cgds->queued;
2674                         cgds->last_reset = tar->last_reset;
2675                         cgds->maxtags = dev->maxtags;
2676                         cgds->mintags = dev->mintags;
2677                         if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2678                                 cgds->last_reset = bus->last_reset;
2679                         mtx_unlock(&devq->send_mtx);
2680                         cgds->ccb_h.status = CAM_REQ_CMP;
2681                 }
2682                 break;
2683         }
2684         case XPT_GDEVLIST:
2685         {
2686                 struct cam_periph       *nperiph;
2687                 struct periph_list      *periph_head;
2688                 struct ccb_getdevlist   *cgdl;
2689                 u_int                   i;
2690                 struct cam_ed           *device;
2691                 int                     found;
2692
2693
2694                 found = 0;
2695
2696                 /*
2697                  * Don't want anyone mucking with our data.
2698                  */
2699                 device = path->device;
2700                 periph_head = &device->periphs;
2701                 cgdl = &start_ccb->cgdl;
2702
2703                 /*
2704                  * Check and see if the list has changed since the user
2705                  * last requested a list member.  If so, tell them that the
2706                  * list has changed, and therefore they need to start over
2707                  * from the beginning.
2708                  */
2709                 if ((cgdl->index != 0) &&
2710                     (cgdl->generation != device->generation)) {
2711                         cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2712                         break;
2713                 }
2714
2715                 /*
2716                  * Traverse the list of peripherals and attempt to find
2717                  * the requested peripheral.
2718                  */
2719                 for (nperiph = SLIST_FIRST(periph_head), i = 0;
2720                      (nperiph != NULL) && (i <= cgdl->index);
2721                      nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2722                         if (i == cgdl->index) {
2723                                 strncpy(cgdl->periph_name,
2724                                         nperiph->periph_name,
2725                                         DEV_IDLEN);
2726                                 cgdl->unit_number = nperiph->unit_number;
2727                                 found = 1;
2728                         }
2729                 }
2730                 if (found == 0) {
2731                         cgdl->status = CAM_GDEVLIST_ERROR;
2732                         break;
2733                 }
2734
2735                 if (nperiph == NULL)
2736                         cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2737                 else
2738                         cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2739
2740                 cgdl->index++;
2741                 cgdl->generation = device->generation;
2742
2743                 cgdl->ccb_h.status = CAM_REQ_CMP;
2744                 break;
2745         }
2746         case XPT_DEV_MATCH:
2747         {
2748                 dev_pos_type position_type;
2749                 struct ccb_dev_match *cdm;
2750
2751                 cdm = &start_ccb->cdm;
2752
2753                 /*
2754                  * There are two ways of getting at information in the EDT.
2755                  * The first way is via the primary EDT tree.  It starts
2756                  * with a list of busses, then a list of targets on a bus,
2757                  * then devices/luns on a target, and then peripherals on a
2758                  * device/lun.  The "other" way is by the peripheral driver
2759                  * lists.  The peripheral driver lists are organized by
2760                  * peripheral driver.  (obviously)  So it makes sense to
2761                  * use the peripheral driver list if the user is looking
2762                  * for something like "da1", or all "da" devices.  If the
2763                  * user is looking for something on a particular bus/target
2764                  * or lun, it's generally better to go through the EDT tree.
2765                  */
2766
2767                 if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2768                         position_type = cdm->pos.position_type;
2769                 else {
2770                         u_int i;
2771
2772                         position_type = CAM_DEV_POS_NONE;
2773
2774                         for (i = 0; i < cdm->num_patterns; i++) {
2775                                 if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2776                                  ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2777                                         position_type = CAM_DEV_POS_EDT;
2778                                         break;
2779                                 }
2780                         }
2781
2782                         if (cdm->num_patterns == 0)
2783                                 position_type = CAM_DEV_POS_EDT;
2784                         else if (position_type == CAM_DEV_POS_NONE)
2785                                 position_type = CAM_DEV_POS_PDRV;
2786                 }
2787
2788                 switch(position_type & CAM_DEV_POS_TYPEMASK) {
2789                 case CAM_DEV_POS_EDT:
2790                         xptedtmatch(cdm);
2791                         break;
2792                 case CAM_DEV_POS_PDRV:
2793                         xptperiphlistmatch(cdm);
2794                         break;
2795                 default:
2796                         cdm->status = CAM_DEV_MATCH_ERROR;
2797                         break;
2798                 }
2799
2800                 if (cdm->status == CAM_DEV_MATCH_ERROR)
2801                         start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2802                 else
2803                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2804
2805                 break;
2806         }
2807         case XPT_SASYNC_CB:
2808         {
2809                 struct ccb_setasync *csa;
2810                 struct async_node *cur_entry;
2811                 struct async_list *async_head;
2812                 u_int32_t added;
2813
2814                 csa = &start_ccb->csa;
2815                 added = csa->event_enable;
2816                 async_head = &path->device->asyncs;
2817
2818                 /*
2819                  * If there is already an entry for us, simply
2820                  * update it.
2821                  */
2822                 cur_entry = SLIST_FIRST(async_head);
2823                 while (cur_entry != NULL) {
2824                         if ((cur_entry->callback_arg == csa->callback_arg)
2825                          && (cur_entry->callback == csa->callback))
2826                                 break;
2827                         cur_entry = SLIST_NEXT(cur_entry, links);
2828                 }
2829
2830                 if (cur_entry != NULL) {
2831                         /*
2832                          * If the request has no flags set,
2833                          * remove the entry.
2834                          */
2835                         added &= ~cur_entry->event_enable;
2836                         if (csa->event_enable == 0) {
2837                                 SLIST_REMOVE(async_head, cur_entry,
2838                                              async_node, links);
2839                                 xpt_release_device(path->device);
2840                                 free(cur_entry, M_CAMXPT);
2841                         } else {
2842                                 cur_entry->event_enable = csa->event_enable;
2843                         }
2844                         csa->event_enable = added;
2845                 } else {
2846                         cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2847                                            M_NOWAIT);
2848                         if (cur_entry == NULL) {
2849                                 csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2850                                 break;
2851                         }
2852                         cur_entry->event_enable = csa->event_enable;
2853                         cur_entry->event_lock =
2854                             mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2855                         cur_entry->callback_arg = csa->callback_arg;
2856                         cur_entry->callback = csa->callback;
2857                         SLIST_INSERT_HEAD(async_head, cur_entry, links);
2858                         xpt_acquire_device(path->device);
2859                 }
2860                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2861                 break;
2862         }
2863         case XPT_REL_SIMQ:
2864         {
2865                 struct ccb_relsim *crs;
2866                 struct cam_ed *dev;
2867
2868                 crs = &start_ccb->crs;
2869                 dev = path->device;
2870                 if (dev == NULL) {
2871
2872                         crs->ccb_h.status = CAM_DEV_NOT_THERE;
2873                         break;
2874                 }
2875
2876                 if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2877
2878                         /* Don't ever go below one opening */
2879                         if (crs->openings > 0) {
2880                                 xpt_dev_ccbq_resize(path, crs->openings);
2881                                 if (bootverbose) {
2882                                         xpt_print(path,
2883                                             "number of openings is now %d\n",
2884                                             crs->openings);
2885                                 }
2886                         }
2887                 }
2888
2889                 mtx_lock(&dev->sim->devq->send_mtx);
2890                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2891
2892                         if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2893
2894                                 /*
2895                                  * Just extend the old timeout and decrement
2896                                  * the freeze count so that a single timeout
2897                                  * is sufficient for releasing the queue.
2898                                  */
2899                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2900                                 callout_stop(&dev->callout);
2901                         } else {
2902
2903                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2904                         }
2905
2906                         callout_reset_sbt(&dev->callout,
2907                             SBT_1MS * crs->release_timeout, 0,
2908                             xpt_release_devq_timeout, dev, 0);
2909
2910                         dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2911
2912                 }
2913
2914                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2915
2916                         if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2917                                 /*
2918                                  * Decrement the freeze count so that a single
2919                                  * completion is still sufficient to unfreeze
2920                                  * the queue.
2921                                  */
2922                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2923                         } else {
2924
2925                                 dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2926                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2927                         }
2928                 }
2929
2930                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2931
2932                         if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2933                          || (dev->ccbq.dev_active == 0)) {
2934
2935                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2936                         } else {
2937
2938                                 dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2939                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2940                         }
2941                 }
2942                 mtx_unlock(&dev->sim->devq->send_mtx);
2943
2944                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2945                         xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2946                 start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2947                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2948                 break;
2949         }
2950         case XPT_DEBUG: {
2951                 struct cam_path *oldpath;
2952
2953                 /* Check that all request bits are supported. */
2954                 if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2955                         start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2956                         break;
2957                 }
2958
2959                 cam_dflags = CAM_DEBUG_NONE;
2960                 if (cam_dpath != NULL) {
2961                         oldpath = cam_dpath;
2962                         cam_dpath = NULL;
2963                         xpt_free_path(oldpath);
2964                 }
2965                 if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2966                         if (xpt_create_path(&cam_dpath, NULL,
2967                                             start_ccb->ccb_h.path_id,
2968                                             start_ccb->ccb_h.target_id,
2969                                             start_ccb->ccb_h.target_lun) !=
2970                                             CAM_REQ_CMP) {
2971                                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2972                         } else {
2973                                 cam_dflags = start_ccb->cdbg.flags;
2974                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2975                                 xpt_print(cam_dpath, "debugging flags now %x\n",
2976                                     cam_dflags);
2977                         }
2978                 } else
2979                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2980                 break;
2981         }
2982         case XPT_NOOP:
2983                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2984                         xpt_freeze_devq(path, 1);
2985                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2986                 break;
2987         default:
2988         case XPT_SDEV_TYPE:
2989         case XPT_TERM_IO:
2990         case XPT_ENG_INQ:
2991                 /* XXX Implement */
2992                 printf("%s: CCB type %#x not supported\n", __func__,
2993                        start_ccb->ccb_h.func_code);
2994                 start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
2995                 if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
2996                         xpt_done(start_ccb);
2997                 }
2998                 break;
2999         }
3000 }
3001
3002 void
3003 xpt_polled_action(union ccb *start_ccb)
3004 {
3005         u_int32_t timeout;
3006         struct    cam_sim *sim;
3007         struct    cam_devq *devq;
3008         struct    cam_ed *dev;
3009
3010         timeout = start_ccb->ccb_h.timeout * 10;
3011         sim = start_ccb->ccb_h.path->bus->sim;
3012         devq = sim->devq;
3013         dev = start_ccb->ccb_h.path->device;
3014
3015         mtx_unlock(&dev->device_mtx);
3016
3017         /*
3018          * Steal an opening so that no other queued requests
3019          * can get it before us while we simulate interrupts.
3020          */
3021         mtx_lock(&devq->send_mtx);
3022         dev->ccbq.dev_openings--;
3023         while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3024             (--timeout > 0)) {
3025                 mtx_unlock(&devq->send_mtx);
3026                 DELAY(100);
3027                 CAM_SIM_LOCK(sim);
3028                 (*(sim->sim_poll))(sim);
3029                 CAM_SIM_UNLOCK(sim);
3030                 camisr_runqueue();
3031                 mtx_lock(&devq->send_mtx);
3032         }
3033         dev->ccbq.dev_openings++;
3034         mtx_unlock(&devq->send_mtx);
3035
3036         if (timeout != 0) {
3037                 xpt_action(start_ccb);
3038                 while(--timeout > 0) {
3039                         CAM_SIM_LOCK(sim);
3040                         (*(sim->sim_poll))(sim);
3041                         CAM_SIM_UNLOCK(sim);
3042                         camisr_runqueue();
3043                         if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3044                             != CAM_REQ_INPROG)
3045                                 break;
3046                         DELAY(100);
3047                 }
3048                 if (timeout == 0) {
3049                         /*
3050                          * XXX Is it worth adding a sim_timeout entry
3051                          * point so we can attempt recovery?  If
3052                          * this is only used for dumps, I don't think
3053                          * it is.
3054                          */
3055                         start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3056                 }
3057         } else {
3058                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3059         }
3060
3061         mtx_lock(&dev->device_mtx);
3062 }
3063
3064 /*
3065  * Schedule a peripheral driver to receive a ccb when its
3066  * target device has space for more transactions.
3067  */
3068 void
3069 xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3070 {
3071
3072         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3073         cam_periph_assert(periph, MA_OWNED);
3074         if (new_priority < periph->scheduled_priority) {
3075                 periph->scheduled_priority = new_priority;
3076                 xpt_run_allocq(periph, 0);
3077         }
3078 }
3079
3080
3081 /*
3082  * Schedule a device to run on a given queue.
3083  * If the device was inserted as a new entry on the queue,
3084  * return 1 meaning the device queue should be run. If we
3085  * were already queued, implying someone else has already
3086  * started the queue, return 0 so the caller doesn't attempt
3087  * to run the queue.
3088  */
3089 static int
3090 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3091                  u_int32_t new_priority)
3092 {
3093         int retval;
3094         u_int32_t old_priority;
3095
3096         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3097
3098         old_priority = pinfo->priority;
3099
3100         /*
3101          * Are we already queued?
3102          */
3103         if (pinfo->index != CAM_UNQUEUED_INDEX) {
3104                 /* Simply reorder based on new priority */
3105                 if (new_priority < old_priority) {
3106                         camq_change_priority(queue, pinfo->index,
3107                                              new_priority);
3108                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3109                                         ("changed priority to %d\n",
3110                                          new_priority));
3111                         retval = 1;
3112                 } else
3113                         retval = 0;
3114         } else {
3115                 /* New entry on the queue */
3116                 if (new_priority < old_priority)
3117                         pinfo->priority = new_priority;
3118
3119                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3120                                 ("Inserting onto queue\n"));
3121                 pinfo->generation = ++queue->generation;
3122                 camq_insert(queue, pinfo);
3123                 retval = 1;
3124         }
3125         return (retval);
3126 }
3127
3128 static void
3129 xpt_run_allocq_task(void *context, int pending)
3130 {
3131         struct cam_periph *periph = context;
3132
3133         cam_periph_lock(periph);
3134         periph->flags &= ~CAM_PERIPH_RUN_TASK;
3135         xpt_run_allocq(periph, 1);
3136         cam_periph_unlock(periph);
3137         cam_periph_release(periph);
3138 }
3139
3140 static void
3141 xpt_run_allocq(struct cam_periph *periph, int sleep)
3142 {
3143         struct cam_ed   *device;
3144         union ccb       *ccb;
3145         uint32_t         prio;
3146
3147         cam_periph_assert(periph, MA_OWNED);
3148         if (periph->periph_allocating)
3149                 return;
3150         periph->periph_allocating = 1;
3151         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3152         device = periph->path->device;
3153         ccb = NULL;
3154 restart:
3155         while ((prio = min(periph->scheduled_priority,
3156             periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3157             (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3158              device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3159
3160                 if (ccb == NULL &&
3161                     (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3162                         if (sleep) {
3163                                 ccb = xpt_get_ccb(periph);
3164                                 goto restart;
3165                         }
3166                         if (periph->flags & CAM_PERIPH_RUN_TASK)
3167                                 break;
3168                         cam_periph_doacquire(periph);
3169                         periph->flags |= CAM_PERIPH_RUN_TASK;
3170                         taskqueue_enqueue(xsoftc.xpt_taskq,
3171                             &periph->periph_run_task);
3172                         break;
3173                 }
3174                 xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3175                 if (prio == periph->immediate_priority) {
3176                         periph->immediate_priority = CAM_PRIORITY_NONE;
3177                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3178                                         ("waking cam_periph_getccb()\n"));
3179                         SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3180                                           periph_links.sle);
3181                         wakeup(&periph->ccb_list);
3182                 } else {
3183                         periph->scheduled_priority = CAM_PRIORITY_NONE;
3184                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3185                                         ("calling periph_start()\n"));
3186                         periph->periph_start(periph, ccb);
3187                 }
3188                 ccb = NULL;
3189         }
3190         if (ccb != NULL)
3191                 xpt_release_ccb(ccb);
3192         periph->periph_allocating = 0;
3193 }
3194
3195 static void
3196 xpt_run_devq(struct cam_devq *devq)
3197 {
3198         char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3199         int lock;
3200
3201         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3202
3203         devq->send_queue.qfrozen_cnt++;
3204         while ((devq->send_queue.entries > 0)
3205             && (devq->send_openings > 0)
3206             && (devq->send_queue.qfrozen_cnt <= 1)) {
3207                 struct  cam_ed *device;
3208                 union ccb *work_ccb;
3209                 struct  cam_sim *sim;
3210
3211                 device = (struct cam_ed *)camq_remove(&devq->send_queue,
3212                                                            CAMQ_HEAD);
3213                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3214                                 ("running device %p\n", device));
3215
3216                 work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3217                 if (work_ccb == NULL) {
3218                         printf("device on run queue with no ccbs???\n");
3219                         continue;
3220                 }
3221
3222                 if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3223
3224                         mtx_lock(&xsoftc.xpt_highpower_lock);
3225                         if (xsoftc.num_highpower <= 0) {
3226                                 /*
3227                                  * We got a high power command, but we
3228                                  * don't have any available slots.  Freeze
3229                                  * the device queue until we have a slot
3230                                  * available.
3231                                  */
3232                                 xpt_freeze_devq_device(device, 1);
3233                                 STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3234                                                    highpowerq_entry);
3235
3236                                 mtx_unlock(&xsoftc.xpt_highpower_lock);
3237                                 continue;
3238                         } else {
3239                                 /*
3240                                  * Consume a high power slot while
3241                                  * this ccb runs.
3242                                  */
3243                                 xsoftc.num_highpower--;
3244                         }
3245                         mtx_unlock(&xsoftc.xpt_highpower_lock);
3246                 }
3247                 cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3248                 cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3249                 devq->send_openings--;
3250                 devq->send_active++;
3251                 xpt_schedule_devq(devq, device);
3252                 mtx_unlock(&devq->send_mtx);
3253
3254                 if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3255                         /*
3256                          * The client wants to freeze the queue
3257                          * after this CCB is sent.
3258                          */
3259                         xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3260                 }
3261
3262                 /* In Target mode, the peripheral driver knows best... */
3263                 if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3264                         if ((device->inq_flags & SID_CmdQue) != 0
3265                          && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3266                                 work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3267                         else
3268                                 /*
3269                                  * Clear this in case of a retried CCB that
3270                                  * failed due to a rejected tag.
3271                                  */
3272                                 work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3273                 }
3274
3275                 switch (work_ccb->ccb_h.func_code) {
3276                 case XPT_SCSI_IO:
3277                         CAM_DEBUG(work_ccb->ccb_h.path,
3278                             CAM_DEBUG_CDB,("%s. CDB: %s\n",
3279                              scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3280                                           &device->inq_data),
3281                              scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3282                                              cdb_str, sizeof(cdb_str))));
3283                         break;
3284                 case XPT_ATA_IO:
3285                         CAM_DEBUG(work_ccb->ccb_h.path,
3286                             CAM_DEBUG_CDB,("%s. ACB: %s\n",
3287                              ata_op_string(&work_ccb->ataio.cmd),
3288                              ata_cmd_string(&work_ccb->ataio.cmd,
3289                                             cdb_str, sizeof(cdb_str))));
3290                         break;
3291                 default:
3292                         break;
3293                 }
3294
3295                 /*
3296                  * Device queues can be shared among multiple SIM instances
3297                  * that reside on different busses.  Use the SIM from the
3298                  * queued device, rather than the one from the calling bus.
3299                  */
3300                 sim = device->sim;
3301                 lock = (mtx_owned(sim->mtx) == 0);
3302                 if (lock)
3303                         CAM_SIM_LOCK(sim);
3304                 (*(sim->sim_action))(sim, work_ccb);
3305                 if (lock)
3306                         CAM_SIM_UNLOCK(sim);
3307                 mtx_lock(&devq->send_mtx);
3308         }
3309         devq->send_queue.qfrozen_cnt--;
3310 }
3311
3312 /*
3313  * This function merges stuff from the slave ccb into the master ccb, while
3314  * keeping important fields in the master ccb constant.
3315  */
3316 void
3317 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3318 {
3319
3320         /*
3321          * Pull fields that are valid for peripheral drivers to set
3322          * into the master CCB along with the CCB "payload".
3323          */
3324         master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3325         master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3326         master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3327         master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3328         bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3329               sizeof(union ccb) - sizeof(struct ccb_hdr));
3330 }
3331
3332 void
3333 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3334 {
3335
3336         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3337         ccb_h->pinfo.priority = priority;
3338         ccb_h->path = path;
3339         ccb_h->path_id = path->bus->path_id;
3340         if (path->target)
3341                 ccb_h->target_id = path->target->target_id;
3342         else
3343                 ccb_h->target_id = CAM_TARGET_WILDCARD;
3344         if (path->device) {
3345                 ccb_h->target_lun = path->device->lun_id;
3346                 ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3347         } else {
3348                 ccb_h->target_lun = CAM_TARGET_WILDCARD;
3349         }
3350         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3351         ccb_h->flags = 0;
3352         ccb_h->xflags = 0;
3353 }
3354
3355 /* Path manipulation functions */
3356 cam_status
3357 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3358                 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3359 {
3360         struct     cam_path *path;
3361         cam_status status;
3362
3363         path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3364
3365         if (path == NULL) {
3366                 status = CAM_RESRC_UNAVAIL;
3367                 return(status);
3368         }
3369         status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3370         if (status != CAM_REQ_CMP) {
3371                 free(path, M_CAMPATH);
3372                 path = NULL;
3373         }
3374         *new_path_ptr = path;
3375         return (status);
3376 }
3377
3378 cam_status
3379 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3380                          struct cam_periph *periph, path_id_t path_id,
3381                          target_id_t target_id, lun_id_t lun_id)
3382 {
3383
3384         return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3385             lun_id));
3386 }
3387
3388 cam_status
3389 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3390                  path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3391 {
3392         struct       cam_eb *bus;
3393         struct       cam_et *target;
3394         struct       cam_ed *device;
3395         cam_status   status;
3396
3397         status = CAM_REQ_CMP;   /* Completed without error */
3398         target = NULL;          /* Wildcarded */
3399         device = NULL;          /* Wildcarded */
3400
3401         /*
3402          * We will potentially modify the EDT, so block interrupts
3403          * that may attempt to create cam paths.
3404          */
3405         bus = xpt_find_bus(path_id);
3406         if (bus == NULL) {
3407                 status = CAM_PATH_INVALID;
3408         } else {
3409                 xpt_lock_buses();
3410                 mtx_lock(&bus->eb_mtx);
3411                 target = xpt_find_target(bus, target_id);
3412                 if (target == NULL) {
3413                         /* Create one */
3414                         struct cam_et *new_target;
3415
3416                         new_target = xpt_alloc_target(bus, target_id);
3417                         if (new_target == NULL) {
3418                                 status = CAM_RESRC_UNAVAIL;
3419                         } else {
3420                                 target = new_target;
3421                         }
3422                 }
3423                 xpt_unlock_buses();
3424                 if (target != NULL) {
3425                         device = xpt_find_device(target, lun_id);
3426                         if (device == NULL) {
3427                                 /* Create one */
3428                                 struct cam_ed *new_device;
3429
3430                                 new_device =
3431                                     (*(bus->xport->alloc_device))(bus,
3432                                                                       target,
3433                                                                       lun_id);
3434                                 if (new_device == NULL) {
3435                                         status = CAM_RESRC_UNAVAIL;
3436                                 } else {
3437                                         device = new_device;
3438                                 }
3439                         }
3440                 }
3441                 mtx_unlock(&bus->eb_mtx);
3442         }
3443
3444         /*
3445          * Only touch the user's data if we are successful.
3446          */
3447         if (status == CAM_REQ_CMP) {
3448                 new_path->periph = perph;
3449                 new_path->bus = bus;
3450                 new_path->target = target;
3451                 new_path->device = device;
3452                 CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3453         } else {
3454                 if (device != NULL)
3455                         xpt_release_device(device);
3456                 if (target != NULL)
3457                         xpt_release_target(target);
3458                 if (bus != NULL)
3459                         xpt_release_bus(bus);
3460         }
3461         return (status);
3462 }
3463
3464 cam_status
3465 xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3466 {
3467         struct     cam_path *new_path;
3468
3469         new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3470         if (new_path == NULL)
3471                 return(CAM_RESRC_UNAVAIL);
3472         xpt_copy_path(new_path, path);
3473         *new_path_ptr = new_path;
3474         return (CAM_REQ_CMP);
3475 }
3476
3477 void
3478 xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3479 {
3480
3481         *new_path = *path;
3482         if (path->bus != NULL)
3483                 xpt_acquire_bus(path->bus);
3484         if (path->target != NULL)
3485                 xpt_acquire_target(path->target);
3486         if (path->device != NULL)
3487                 xpt_acquire_device(path->device);
3488 }
3489
3490 void
3491 xpt_release_path(struct cam_path *path)
3492 {
3493         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3494         if (path->device != NULL) {
3495                 xpt_release_device(path->device);
3496                 path->device = NULL;
3497         }
3498         if (path->target != NULL) {
3499                 xpt_release_target(path->target);
3500                 path->target = NULL;
3501         }
3502         if (path->bus != NULL) {
3503                 xpt_release_bus(path->bus);
3504                 path->bus = NULL;
3505         }
3506 }
3507
3508 void
3509 xpt_free_path(struct cam_path *path)
3510 {
3511
3512         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3513         xpt_release_path(path);
3514         free(path, M_CAMPATH);
3515 }
3516
3517 void
3518 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3519     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3520 {
3521
3522         xpt_lock_buses();
3523         if (bus_ref) {
3524                 if (path->bus)
3525                         *bus_ref = path->bus->refcount;
3526                 else
3527                         *bus_ref = 0;
3528         }
3529         if (periph_ref) {
3530                 if (path->periph)
3531                         *periph_ref = path->periph->refcount;
3532                 else
3533                         *periph_ref = 0;
3534         }
3535         xpt_unlock_buses();
3536         if (target_ref) {
3537                 if (path->target)
3538                         *target_ref = path->target->refcount;
3539                 else
3540                         *target_ref = 0;
3541         }
3542         if (device_ref) {
3543                 if (path->device)
3544                         *device_ref = path->device->refcount;
3545                 else
3546                         *device_ref = 0;
3547         }
3548 }
3549
3550 /*
3551  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3552  * in path1, 2 for match with wildcards in path2.
3553  */
3554 int
3555 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3556 {
3557         int retval = 0;
3558
3559         if (path1->bus != path2->bus) {
3560                 if (path1->bus->path_id == CAM_BUS_WILDCARD)
3561                         retval = 1;
3562                 else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3563                         retval = 2;
3564                 else
3565                         return (-1);
3566         }
3567         if (path1->target != path2->target) {
3568                 if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3569                         if (retval == 0)
3570                                 retval = 1;
3571                 } else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3572                         retval = 2;
3573                 else
3574                         return (-1);
3575         }
3576         if (path1->device != path2->device) {
3577                 if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3578                         if (retval == 0)
3579                                 retval = 1;
3580                 } else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3581                         retval = 2;
3582                 else
3583                         return (-1);
3584         }
3585         return (retval);
3586 }
3587
3588 int
3589 xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3590 {
3591         int retval = 0;
3592
3593         if (path->bus != dev->target->bus) {
3594                 if (path->bus->path_id == CAM_BUS_WILDCARD)
3595                         retval = 1;
3596                 else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3597                         retval = 2;
3598                 else
3599                         return (-1);
3600         }
3601         if (path->target != dev->target) {
3602                 if (path->target->target_id == CAM_TARGET_WILDCARD) {
3603                         if (retval == 0)
3604                                 retval = 1;
3605                 } else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3606                         retval = 2;
3607                 else
3608                         return (-1);
3609         }
3610         if (path->device != dev) {
3611                 if (path->device->lun_id == CAM_LUN_WILDCARD) {
3612                         if (retval == 0)
3613                                 retval = 1;
3614                 } else if (dev->lun_id == CAM_LUN_WILDCARD)
3615                         retval = 2;
3616                 else
3617                         return (-1);
3618         }
3619         return (retval);
3620 }
3621
3622 void
3623 xpt_print_path(struct cam_path *path)
3624 {
3625
3626         if (path == NULL)
3627                 printf("(nopath): ");
3628         else {
3629                 if (path->periph != NULL)
3630                         printf("(%s%d:", path->periph->periph_name,
3631                                path->periph->unit_number);
3632                 else
3633                         printf("(noperiph:");
3634
3635                 if (path->bus != NULL)
3636                         printf("%s%d:%d:", path->bus->sim->sim_name,
3637                                path->bus->sim->unit_number,
3638                                path->bus->sim->bus_id);
3639                 else
3640                         printf("nobus:");
3641
3642                 if (path->target != NULL)
3643                         printf("%d:", path->target->target_id);
3644                 else
3645                         printf("X:");
3646
3647                 if (path->device != NULL)
3648                         printf("%jx): ", (uintmax_t)path->device->lun_id);
3649                 else
3650                         printf("X): ");
3651         }
3652 }
3653
3654 void
3655 xpt_print_device(struct cam_ed *device)
3656 {
3657
3658         if (device == NULL)
3659                 printf("(nopath): ");
3660         else {
3661                 printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3662                        device->sim->unit_number,
3663                        device->sim->bus_id,
3664                        device->target->target_id,
3665                        (uintmax_t)device->lun_id);
3666         }
3667 }
3668
3669 void
3670 xpt_print(struct cam_path *path, const char *fmt, ...)
3671 {
3672         va_list ap;
3673         xpt_print_path(path);
3674         va_start(ap, fmt);
3675         vprintf(fmt, ap);
3676         va_end(ap);
3677 }
3678
3679 int
3680 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3681 {
3682         struct sbuf sb;
3683
3684         sbuf_new(&sb, str, str_len, 0);
3685
3686         if (path == NULL)
3687                 sbuf_printf(&sb, "(nopath): ");
3688         else {
3689                 if (path->periph != NULL)
3690                         sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3691                                     path->periph->unit_number);
3692                 else
3693                         sbuf_printf(&sb, "(noperiph:");
3694
3695                 if (path->bus != NULL)
3696                         sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3697                                     path->bus->sim->unit_number,
3698                                     path->bus->sim->bus_id);
3699                 else
3700                         sbuf_printf(&sb, "nobus:");
3701
3702                 if (path->target != NULL)
3703                         sbuf_printf(&sb, "%d:", path->target->target_id);
3704                 else
3705                         sbuf_printf(&sb, "X:");
3706
3707                 if (path->device != NULL)
3708                         sbuf_printf(&sb, "%jx): ",
3709                             (uintmax_t)path->device->lun_id);
3710                 else
3711                         sbuf_printf(&sb, "X): ");
3712         }
3713         sbuf_finish(&sb);
3714
3715         return(sbuf_len(&sb));
3716 }
3717
3718 path_id_t
3719 xpt_path_path_id(struct cam_path *path)
3720 {
3721         return(path->bus->path_id);
3722 }
3723
3724 target_id_t
3725 xpt_path_target_id(struct cam_path *path)
3726 {
3727         if (path->target != NULL)
3728                 return (path->target->target_id);
3729         else
3730                 return (CAM_TARGET_WILDCARD);
3731 }
3732
3733 lun_id_t
3734 xpt_path_lun_id(struct cam_path *path)
3735 {
3736         if (path->device != NULL)
3737                 return (path->device->lun_id);
3738         else
3739                 return (CAM_LUN_WILDCARD);
3740 }
3741
3742 struct cam_sim *
3743 xpt_path_sim(struct cam_path *path)
3744 {
3745
3746         return (path->bus->sim);
3747 }
3748
3749 struct cam_periph*
3750 xpt_path_periph(struct cam_path *path)
3751 {
3752
3753         return (path->periph);
3754 }
3755
3756 int
3757 xpt_path_legacy_ata_id(struct cam_path *path)
3758 {
3759         struct cam_eb *bus;
3760         int bus_id;
3761
3762         if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3763             strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3764             strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3765             strcmp(path->bus->sim->sim_name, "siisch") != 0)
3766                 return (-1);
3767
3768         if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3769             path->bus->sim->unit_number < 2) {
3770                 bus_id = path->bus->sim->unit_number;
3771         } else {
3772                 bus_id = 2;
3773                 xpt_lock_buses();
3774                 TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3775                         if (bus == path->bus)
3776                                 break;
3777                         if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3778                              bus->sim->unit_number >= 2) ||
3779                             strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3780                             strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3781                             strcmp(bus->sim->sim_name, "siisch") == 0)
3782                                 bus_id++;
3783                 }
3784                 xpt_unlock_buses();
3785         }
3786         if (path->target != NULL) {
3787                 if (path->target->target_id < 2)
3788                         return (bus_id * 2 + path->target->target_id);
3789                 else
3790                         return (-1);
3791         } else
3792                 return (bus_id * 2);
3793 }
3794
3795 /*
3796  * Release a CAM control block for the caller.  Remit the cost of the structure
3797  * to the device referenced by the path.  If the this device had no 'credits'
3798  * and peripheral drivers have registered async callbacks for this notification
3799  * call them now.
3800  */
3801 void
3802 xpt_release_ccb(union ccb *free_ccb)
3803 {
3804         struct   cam_ed *device;
3805         struct   cam_periph *periph;
3806
3807         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3808         xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3809         device = free_ccb->ccb_h.path->device;
3810         periph = free_ccb->ccb_h.path->periph;
3811
3812         xpt_free_ccb(free_ccb);
3813         periph->periph_allocated--;
3814         cam_ccbq_release_opening(&device->ccbq);
3815         xpt_run_allocq(periph, 0);
3816 }
3817
3818 /* Functions accessed by SIM drivers */
3819
3820 static struct xpt_xport xport_default = {
3821         .alloc_device = xpt_alloc_device_default,
3822         .action = xpt_action_default,
3823         .async = xpt_dev_async_default,
3824 };
3825
3826 /*
3827  * A sim structure, listing the SIM entry points and instance
3828  * identification info is passed to xpt_bus_register to hook the SIM
3829  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3830  * for this new bus and places it in the array of busses and assigns
3831  * it a path_id.  The path_id may be influenced by "hard wiring"
3832  * information specified by the user.  Once interrupt services are
3833  * available, the bus will be probed.
3834  */
3835 int32_t
3836 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3837 {
3838         struct cam_eb *new_bus;
3839         struct cam_eb *old_bus;
3840         struct ccb_pathinq cpi;
3841         struct cam_path *path;
3842         cam_status status;
3843
3844         mtx_assert(sim->mtx, MA_OWNED);
3845
3846         sim->bus_id = bus;
3847         new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3848                                           M_CAMXPT, M_NOWAIT|M_ZERO);
3849         if (new_bus == NULL) {
3850                 /* Couldn't satisfy request */
3851                 return (CAM_RESRC_UNAVAIL);
3852         }
3853
3854         mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3855         TAILQ_INIT(&new_bus->et_entries);
3856         cam_sim_hold(sim);
3857         new_bus->sim = sim;
3858         timevalclear(&new_bus->last_reset);
3859         new_bus->flags = 0;
3860         new_bus->refcount = 1;  /* Held until a bus_deregister event */
3861         new_bus->generation = 0;
3862
3863         xpt_lock_buses();
3864         sim->path_id = new_bus->path_id =
3865             xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3866         old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3867         while (old_bus != NULL
3868             && old_bus->path_id < new_bus->path_id)
3869                 old_bus = TAILQ_NEXT(old_bus, links);
3870         if (old_bus != NULL)
3871                 TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3872         else
3873                 TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3874         xsoftc.bus_generation++;
3875         xpt_unlock_buses();
3876
3877         /*
3878          * Set a default transport so that a PATH_INQ can be issued to
3879          * the SIM.  This will then allow for probing and attaching of
3880          * a more appropriate transport.
3881          */
3882         new_bus->xport = &xport_default;
3883
3884         status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3885                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3886         if (status != CAM_REQ_CMP) {
3887                 xpt_release_bus(new_bus);
3888                 free(path, M_CAMXPT);
3889                 return (CAM_RESRC_UNAVAIL);
3890         }
3891
3892         xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3893         cpi.ccb_h.func_code = XPT_PATH_INQ;
3894         xpt_action((union ccb *)&cpi);
3895
3896         if (cpi.ccb_h.status == CAM_REQ_CMP) {
3897                 switch (cpi.transport) {
3898                 case XPORT_SPI:
3899                 case XPORT_SAS:
3900                 case XPORT_FC:
3901                 case XPORT_USB:
3902                 case XPORT_ISCSI:
3903                 case XPORT_SRP:
3904                 case XPORT_PPB:
3905                         new_bus->xport = scsi_get_xport();
3906                         break;
3907                 case XPORT_ATA:
3908                 case XPORT_SATA:
3909                         new_bus->xport = ata_get_xport();
3910                         break;
3911                 default:
3912                         new_bus->xport = &xport_default;
3913                         break;
3914                 }
3915         }
3916
3917         /* Notify interested parties */
3918         if (sim->path_id != CAM_XPT_PATH_ID) {
3919
3920                 xpt_async(AC_PATH_REGISTERED, path, &cpi);
3921                 if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3922                         union   ccb *scan_ccb;
3923
3924                         /* Initiate bus rescan. */
3925                         scan_ccb = xpt_alloc_ccb_nowait();
3926                         if (scan_ccb != NULL) {
3927                                 scan_ccb->ccb_h.path = path;
3928                                 scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3929                                 scan_ccb->crcn.flags = 0;
3930                                 xpt_rescan(scan_ccb);
3931                         } else {
3932                                 xpt_print(path,
3933                                           "Can't allocate CCB to scan bus\n");
3934                                 xpt_free_path(path);
3935                         }
3936                 } else
3937                         xpt_free_path(path);
3938         } else
3939                 xpt_free_path(path);
3940         return (CAM_SUCCESS);
3941 }
3942
3943 int32_t
3944 xpt_bus_deregister(path_id_t pathid)
3945 {
3946         struct cam_path bus_path;
3947         cam_status status;
3948
3949         status = xpt_compile_path(&bus_path, NULL, pathid,
3950                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3951         if (status != CAM_REQ_CMP)
3952                 return (status);
3953
3954         xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3955         xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3956
3957         /* Release the reference count held while registered. */
3958         xpt_release_bus(bus_path.bus);
3959         xpt_release_path(&bus_path);
3960
3961         return (CAM_REQ_CMP);
3962 }
3963
3964 static path_id_t
3965 xptnextfreepathid(void)
3966 {
3967         struct cam_eb *bus;
3968         path_id_t pathid;
3969         const char *strval;
3970
3971         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3972         pathid = 0;
3973         bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3974 retry:
3975         /* Find an unoccupied pathid */
3976         while (bus != NULL && bus->path_id <= pathid) {
3977                 if (bus->path_id == pathid)
3978                         pathid++;
3979                 bus = TAILQ_NEXT(bus, links);
3980         }
3981
3982         /*
3983          * Ensure that this pathid is not reserved for
3984          * a bus that may be registered in the future.
3985          */
3986         if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
3987                 ++pathid;
3988                 /* Start the search over */
3989                 goto retry;
3990         }
3991         return (pathid);
3992 }
3993
3994 static path_id_t
3995 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
3996 {
3997         path_id_t pathid;
3998         int i, dunit, val;
3999         char buf[32];
4000         const char *dname;
4001
4002         pathid = CAM_XPT_PATH_ID;
4003         snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4004         if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4005                 return (pathid);
4006         i = 0;
4007         while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4008                 if (strcmp(dname, "scbus")) {
4009                         /* Avoid a bit of foot shooting. */
4010                         continue;
4011                 }
4012                 if (dunit < 0)          /* unwired?! */
4013                         continue;
4014                 if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4015                         if (sim_bus == val) {
4016                                 pathid = dunit;
4017                                 break;
4018                         }
4019                 } else if (sim_bus == 0) {
4020                         /* Unspecified matches bus 0 */
4021                         pathid = dunit;
4022                         break;
4023                 } else {
4024                         printf("Ambiguous scbus configuration for %s%d "
4025                                "bus %d, cannot wire down.  The kernel "
4026                                "config entry for scbus%d should "
4027                                "specify a controller bus.\n"
4028                                "Scbus will be assigned dynamically.\n",
4029                                sim_name, sim_unit, sim_bus, dunit);
4030                         break;
4031                 }
4032         }
4033
4034         if (pathid == CAM_XPT_PATH_ID)
4035                 pathid = xptnextfreepathid();
4036         return (pathid);
4037 }
4038
4039 static const char *
4040 xpt_async_string(u_int32_t async_code)
4041 {
4042
4043         switch (async_code) {
4044         case AC_BUS_RESET: return ("AC_BUS_RESET");
4045         case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4046         case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4047         case AC_SENT_BDR: return ("AC_SENT_BDR");
4048         case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4049         case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4050         case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4051         case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4052         case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4053         case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4054         case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4055         case AC_CONTRACT: return ("AC_CONTRACT");
4056         case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4057         case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4058         }
4059         return ("AC_UNKNOWN");
4060 }
4061
4062 static int
4063 xpt_async_size(u_int32_t async_code)
4064 {
4065
4066         switch (async_code) {
4067         case AC_BUS_RESET: return (0);
4068         case AC_UNSOL_RESEL: return (0);
4069         case AC_SCSI_AEN: return (0);
4070         case AC_SENT_BDR: return (0);
4071         case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4072         case AC_PATH_DEREGISTERED: return (0);
4073         case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4074         case AC_LOST_DEVICE: return (0);
4075         case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4076         case AC_INQ_CHANGED: return (0);
4077         case AC_GETDEV_CHANGED: return (0);
4078         case AC_CONTRACT: return (sizeof(struct ac_contract));
4079         case AC_ADVINFO_CHANGED: return (-1);
4080         case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4081         }
4082         return (0);
4083 }
4084
4085 static int
4086 xpt_async_process_dev(struct cam_ed *device, void *arg)
4087 {
4088         union ccb *ccb = arg;
4089         struct cam_path *path = ccb->ccb_h.path;
4090         void *async_arg = ccb->casync.async_arg_ptr;
4091         u_int32_t async_code = ccb->casync.async_code;
4092         int relock;
4093
4094         if (path->device != device
4095          && path->device->lun_id != CAM_LUN_WILDCARD
4096          && device->lun_id != CAM_LUN_WILDCARD)
4097                 return (1);
4098
4099         /*
4100          * The async callback could free the device.
4101          * If it is a broadcast async, it doesn't hold
4102          * device reference, so take our own reference.
4103          */
4104         xpt_acquire_device(device);
4105
4106         /*
4107          * If async for specific device is to be delivered to
4108          * the wildcard client, take the specific device lock.
4109          * XXX: We may need a way for client to specify it.
4110          */
4111         if ((device->lun_id == CAM_LUN_WILDCARD &&
4112              path->device->lun_id != CAM_LUN_WILDCARD) ||
4113             (device->target->target_id == CAM_TARGET_WILDCARD &&
4114              path->target->target_id != CAM_TARGET_WILDCARD) ||
4115             (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4116              path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4117                 mtx_unlock(&device->device_mtx);
4118                 xpt_path_lock(path);
4119                 relock = 1;
4120         } else
4121                 relock = 0;
4122
4123         (*(device->target->bus->xport->async))(async_code,
4124             device->target->bus, device->target, device, async_arg);
4125         xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4126
4127         if (relock) {
4128                 xpt_path_unlock(path);
4129                 mtx_lock(&device->device_mtx);
4130         }
4131         xpt_release_device(device);
4132         return (1);
4133 }
4134
4135 static int
4136 xpt_async_process_tgt(struct cam_et *target, void *arg)
4137 {
4138         union ccb *ccb = arg;
4139         struct cam_path *path = ccb->ccb_h.path;
4140
4141         if (path->target != target
4142          && path->target->target_id != CAM_TARGET_WILDCARD
4143          && target->target_id != CAM_TARGET_WILDCARD)
4144                 return (1);
4145
4146         if (ccb->casync.async_code == AC_SENT_BDR) {
4147                 /* Update our notion of when the last reset occurred */
4148                 microtime(&target->last_reset);
4149         }
4150
4151         return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4152 }
4153
4154 static void
4155 xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4156 {
4157         struct cam_eb *bus;
4158         struct cam_path *path;
4159         void *async_arg;
4160         u_int32_t async_code;
4161
4162         path = ccb->ccb_h.path;
4163         async_code = ccb->casync.async_code;
4164         async_arg = ccb->casync.async_arg_ptr;
4165         CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4166             ("xpt_async(%s)\n", xpt_async_string(async_code)));
4167         bus = path->bus;
4168
4169         if (async_code == AC_BUS_RESET) {
4170                 /* Update our notion of when the last reset occurred */
4171                 microtime(&bus->last_reset);
4172         }
4173
4174         xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4175
4176         /*
4177          * If this wasn't a fully wildcarded async, tell all
4178          * clients that want all async events.
4179          */
4180         if (bus != xpt_periph->path->bus) {
4181                 xpt_path_lock(xpt_periph->path);
4182                 xpt_async_process_dev(xpt_periph->path->device, ccb);
4183                 xpt_path_unlock(xpt_periph->path);
4184         }
4185
4186         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4187                 xpt_release_devq(path, 1, TRUE);
4188         else
4189                 xpt_release_simq(path->bus->sim, TRUE);
4190         if (ccb->casync.async_arg_size > 0)
4191                 free(async_arg, M_CAMXPT);
4192         xpt_free_path(path);
4193         xpt_free_ccb(ccb);
4194 }
4195
4196 static void
4197 xpt_async_bcast(struct async_list *async_head,
4198                 u_int32_t async_code,
4199                 struct cam_path *path, void *async_arg)
4200 {
4201         struct async_node *cur_entry;
4202         int lock;
4203
4204         cur_entry = SLIST_FIRST(async_head);
4205         while (cur_entry != NULL) {
4206                 struct async_node *next_entry;
4207                 /*
4208                  * Grab the next list entry before we call the current
4209                  * entry's callback.  This is because the callback function
4210                  * can delete its async callback entry.
4211                  */
4212                 next_entry = SLIST_NEXT(cur_entry, links);
4213                 if ((cur_entry->event_enable & async_code) != 0) {
4214                         lock = cur_entry->event_lock;
4215                         if (lock)
4216                                 CAM_SIM_LOCK(path->device->sim);
4217                         cur_entry->callback(cur_entry->callback_arg,
4218                                             async_code, path,
4219                                             async_arg);
4220                         if (lock)
4221                                 CAM_SIM_UNLOCK(path->device->sim);
4222                 }
4223                 cur_entry = next_entry;
4224         }
4225 }
4226
4227 void
4228 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4229 {
4230         union ccb *ccb;
4231         int size;
4232
4233         ccb = xpt_alloc_ccb_nowait();
4234         if (ccb == NULL) {
4235                 xpt_print(path, "Can't allocate CCB to send %s\n",
4236                     xpt_async_string(async_code));
4237                 return;
4238         }
4239
4240         if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4241                 xpt_print(path, "Can't allocate path to send %s\n",
4242                     xpt_async_string(async_code));
4243                 xpt_free_ccb(ccb);
4244                 return;
4245         }
4246         ccb->ccb_h.path->periph = NULL;
4247         ccb->ccb_h.func_code = XPT_ASYNC;
4248         ccb->ccb_h.cbfcnp = xpt_async_process;
4249         ccb->ccb_h.flags |= CAM_UNLOCKED;
4250         ccb->casync.async_code = async_code;
4251         ccb->casync.async_arg_size = 0;
4252         size = xpt_async_size(async_code);
4253         if (size > 0 && async_arg != NULL) {
4254                 ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4255                 if (ccb->casync.async_arg_ptr == NULL) {
4256                         xpt_print(path, "Can't allocate argument to send %s\n",
4257                             xpt_async_string(async_code));
4258                         xpt_free_path(ccb->ccb_h.path);
4259                         xpt_free_ccb(ccb);
4260                         return;
4261                 }
4262                 memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4263                 ccb->casync.async_arg_size = size;
4264         } else if (size < 0)
4265                 ccb->casync.async_arg_size = size;
4266         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4267                 xpt_freeze_devq(path, 1);
4268         else
4269                 xpt_freeze_simq(path->bus->sim, 1);
4270         xpt_done(ccb);
4271 }
4272
4273 static void
4274 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4275                       struct cam_et *target, struct cam_ed *device,
4276                       void *async_arg)
4277 {
4278
4279         /*
4280          * We only need to handle events for real devices.
4281          */
4282         if (target->target_id == CAM_TARGET_WILDCARD
4283          || device->lun_id == CAM_LUN_WILDCARD)
4284                 return;
4285
4286         printf("%s called\n", __func__);
4287 }
4288
4289 static uint32_t
4290 xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4291 {
4292         struct cam_devq *devq;
4293         uint32_t freeze;
4294
4295         devq = dev->sim->devq;
4296         mtx_assert(&devq->send_mtx, MA_OWNED);
4297         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4298             ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4299             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4300         freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4301         /* Remove frozen device from sendq. */
4302         if (device_is_queued(dev))
4303                 camq_remove(&devq->send_queue, dev->devq_entry.index);
4304         return (freeze);
4305 }
4306
4307 u_int32_t
4308 xpt_freeze_devq(struct cam_path *path, u_int count)
4309 {
4310         struct cam_ed   *dev = path->device;
4311         struct cam_devq *devq;
4312         uint32_t         freeze;
4313
4314         devq = dev->sim->devq;
4315         mtx_lock(&devq->send_mtx);
4316         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4317         freeze = xpt_freeze_devq_device(dev, count);
4318         mtx_unlock(&devq->send_mtx);
4319         return (freeze);
4320 }
4321
4322 u_int32_t
4323 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4324 {
4325         struct cam_devq *devq;
4326         uint32_t         freeze;
4327
4328         devq = sim->devq;
4329         mtx_lock(&devq->send_mtx);
4330         freeze = (devq->send_queue.qfrozen_cnt += count);
4331         mtx_unlock(&devq->send_mtx);
4332         return (freeze);
4333 }
4334
4335 static void
4336 xpt_release_devq_timeout(void *arg)
4337 {
4338         struct cam_ed *dev;
4339         struct cam_devq *devq;
4340
4341         dev = (struct cam_ed *)arg;
4342         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4343         devq = dev->sim->devq;
4344         mtx_assert(&devq->send_mtx, MA_OWNED);
4345         if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4346                 xpt_run_devq(devq);
4347 }
4348
4349 void
4350 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4351 {
4352         struct cam_ed *dev;
4353         struct cam_devq *devq;
4354
4355         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4356             count, run_queue));
4357         dev = path->device;
4358         devq = dev->sim->devq;
4359         mtx_lock(&devq->send_mtx);
4360         if (xpt_release_devq_device(dev, count, run_queue))
4361                 xpt_run_devq(dev->sim->devq);
4362         mtx_unlock(&devq->send_mtx);
4363 }
4364
4365 static int
4366 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4367 {
4368
4369         mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4370         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4371             ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4372             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4373         if (count > dev->ccbq.queue.qfrozen_cnt) {
4374 #ifdef INVARIANTS
4375                 printf("xpt_release_devq(): requested %u > present %u\n",
4376                     count, dev->ccbq.queue.qfrozen_cnt);
4377 #endif
4378                 count = dev->ccbq.queue.qfrozen_cnt;
4379         }
4380         dev->ccbq.queue.qfrozen_cnt -= count;
4381         if (dev->ccbq.queue.qfrozen_cnt == 0) {
4382                 /*
4383                  * No longer need to wait for a successful
4384                  * command completion.
4385                  */
4386                 dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4387                 /*
4388                  * Remove any timeouts that might be scheduled
4389                  * to release this queue.
4390                  */
4391                 if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4392                         callout_stop(&dev->callout);
4393                         dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4394                 }
4395                 /*
4396                  * Now that we are unfrozen schedule the
4397                  * device so any pending transactions are
4398                  * run.
4399                  */
4400                 xpt_schedule_devq(dev->sim->devq, dev);
4401         } else
4402                 run_queue = 0;
4403         return (run_queue);
4404 }
4405
4406 void
4407 xpt_release_simq(struct cam_sim *sim, int run_queue)
4408 {
4409         struct cam_devq *devq;
4410
4411         devq = sim->devq;
4412         mtx_lock(&devq->send_mtx);
4413         if (devq->send_queue.qfrozen_cnt <= 0) {
4414 #ifdef INVARIANTS
4415                 printf("xpt_release_simq: requested 1 > present %u\n",
4416                     devq->send_queue.qfrozen_cnt);
4417 #endif
4418         } else
4419                 devq->send_queue.qfrozen_cnt--;
4420         if (devq->send_queue.qfrozen_cnt == 0) {
4421                 /*
4422                  * If there is a timeout scheduled to release this
4423                  * sim queue, remove it.  The queue frozen count is
4424                  * already at 0.
4425                  */
4426                 if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4427                         callout_stop(&sim->callout);
4428                         sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4429                 }
4430                 if (run_queue) {
4431                         /*
4432                          * Now that we are unfrozen run the send queue.
4433                          */
4434                         xpt_run_devq(sim->devq);
4435                 }
4436         }
4437         mtx_unlock(&devq->send_mtx);
4438 }
4439
4440 /*
4441  * XXX Appears to be unused.
4442  */
4443 static void
4444 xpt_release_simq_timeout(void *arg)
4445 {
4446         struct cam_sim *sim;
4447
4448         sim = (struct cam_sim *)arg;
4449         xpt_release_simq(sim, /* run_queue */ TRUE);
4450 }
4451
4452 void
4453 xpt_done(union ccb *done_ccb)
4454 {
4455         struct cam_doneq *queue;
4456         int     run, hash;
4457
4458         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4459         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4460                 return;
4461
4462         hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4463             done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4464         queue = &cam_doneqs[hash];
4465         mtx_lock(&queue->cam_doneq_mtx);
4466         run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4467         STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4468         done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4469         mtx_unlock(&queue->cam_doneq_mtx);
4470         if (run)
4471                 wakeup(&queue->cam_doneq);
4472 }
4473
4474 void
4475 xpt_done_direct(union ccb *done_ccb)
4476 {
4477
4478         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4479         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4480                 return;
4481
4482         xpt_done_process(&done_ccb->ccb_h);
4483 }
4484
4485 union ccb *
4486 xpt_alloc_ccb()
4487 {
4488         union ccb *new_ccb;
4489
4490         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4491         return (new_ccb);
4492 }
4493
4494 union ccb *
4495 xpt_alloc_ccb_nowait()
4496 {
4497         union ccb *new_ccb;
4498
4499         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4500         return (new_ccb);
4501 }
4502
4503 void
4504 xpt_free_ccb(union ccb *free_ccb)
4505 {
4506         free(free_ccb, M_CAMCCB);
4507 }
4508
4509
4510
4511 /* Private XPT functions */
4512
4513 /*
4514  * Get a CAM control block for the caller. Charge the structure to the device
4515  * referenced by the path.  If we don't have sufficient resources to allocate
4516  * more ccbs, we return NULL.
4517  */
4518 static union ccb *
4519 xpt_get_ccb_nowait(struct cam_periph *periph)
4520 {
4521         union ccb *new_ccb;
4522
4523         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_NOWAIT);
4524         if (new_ccb == NULL)
4525                 return (NULL);
4526         periph->periph_allocated++;
4527         cam_ccbq_take_opening(&periph->path->device->ccbq);
4528         return (new_ccb);
4529 }
4530
4531 static union ccb *
4532 xpt_get_ccb(struct cam_periph *periph)
4533 {
4534         union ccb *new_ccb;
4535
4536         cam_periph_unlock(periph);
4537         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_WAITOK);
4538         cam_periph_lock(periph);
4539         periph->periph_allocated++;
4540         cam_ccbq_take_opening(&periph->path->device->ccbq);
4541         return (new_ccb);
4542 }
4543
4544 union ccb *
4545 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4546 {
4547         struct ccb_hdr *ccb_h;
4548
4549         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4550         cam_periph_assert(periph, MA_OWNED);
4551         while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4552             ccb_h->pinfo.priority != priority) {
4553                 if (priority < periph->immediate_priority) {
4554                         periph->immediate_priority = priority;
4555                         xpt_run_allocq(periph, 0);
4556                 } else
4557                         cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4558                             "cgticb", 0);
4559         }
4560         SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4561         return ((union ccb *)ccb_h);
4562 }
4563
4564 static void
4565 xpt_acquire_bus(struct cam_eb *bus)
4566 {
4567
4568         xpt_lock_buses();
4569         bus->refcount++;
4570         xpt_unlock_buses();
4571 }
4572
4573 static void
4574 xpt_release_bus(struct cam_eb *bus)
4575 {
4576
4577         xpt_lock_buses();
4578         KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4579         if (--bus->refcount > 0) {
4580                 xpt_unlock_buses();
4581                 return;
4582         }
4583         TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4584         xsoftc.bus_generation++;
4585         xpt_unlock_buses();
4586         KASSERT(TAILQ_EMPTY(&bus->et_entries),
4587             ("destroying bus, but target list is not empty"));
4588         cam_sim_release(bus->sim);
4589         mtx_destroy(&bus->eb_mtx);
4590         free(bus, M_CAMXPT);
4591 }
4592
4593 static struct cam_et *
4594 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4595 {
4596         struct cam_et *cur_target, *target;
4597
4598         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4599         mtx_assert(&bus->eb_mtx, MA_OWNED);
4600         target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4601                                          M_NOWAIT|M_ZERO);
4602         if (target == NULL)
4603                 return (NULL);
4604
4605         TAILQ_INIT(&target->ed_entries);
4606         target->bus = bus;
4607         target->target_id = target_id;
4608         target->refcount = 1;
4609         target->generation = 0;
4610         target->luns = NULL;
4611         mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4612         timevalclear(&target->last_reset);
4613         /*
4614          * Hold a reference to our parent bus so it
4615          * will not go away before we do.
4616          */
4617         bus->refcount++;
4618
4619         /* Insertion sort into our bus's target list */
4620         cur_target = TAILQ_FIRST(&bus->et_entries);
4621         while (cur_target != NULL && cur_target->target_id < target_id)
4622                 cur_target = TAILQ_NEXT(cur_target, links);
4623         if (cur_target != NULL) {
4624                 TAILQ_INSERT_BEFORE(cur_target, target, links);
4625         } else {
4626                 TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4627         }
4628         bus->generation++;
4629         return (target);
4630 }
4631
4632 static void
4633 xpt_acquire_target(struct cam_et *target)
4634 {
4635         struct cam_eb *bus = target->bus;
4636
4637         mtx_lock(&bus->eb_mtx);
4638         target->refcount++;
4639         mtx_unlock(&bus->eb_mtx);
4640 }
4641
4642 static void
4643 xpt_release_target(struct cam_et *target)
4644 {
4645         struct cam_eb *bus = target->bus;
4646
4647         mtx_lock(&bus->eb_mtx);
4648         if (--target->refcount > 0) {
4649                 mtx_unlock(&bus->eb_mtx);
4650                 return;
4651         }
4652         TAILQ_REMOVE(&bus->et_entries, target, links);
4653         bus->generation++;
4654         mtx_unlock(&bus->eb_mtx);
4655         KASSERT(TAILQ_EMPTY(&target->ed_entries),
4656             ("destroying target, but device list is not empty"));
4657         xpt_release_bus(bus);
4658         mtx_destroy(&target->luns_mtx);
4659         if (target->luns)
4660                 free(target->luns, M_CAMXPT);
4661         free(target, M_CAMXPT);
4662 }
4663
4664 static struct cam_ed *
4665 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4666                          lun_id_t lun_id)
4667 {
4668         struct cam_ed *device;
4669
4670         device = xpt_alloc_device(bus, target, lun_id);
4671         if (device == NULL)
4672                 return (NULL);
4673
4674         device->mintags = 1;
4675         device->maxtags = 1;
4676         return (device);
4677 }
4678
4679 static void
4680 xpt_destroy_device(void *context, int pending)
4681 {
4682         struct cam_ed   *device = context;
4683
4684         mtx_lock(&device->device_mtx);
4685         mtx_destroy(&device->device_mtx);
4686         free(device, M_CAMDEV);
4687 }
4688
4689 struct cam_ed *
4690 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4691 {
4692         struct cam_ed   *cur_device, *device;
4693         struct cam_devq *devq;
4694         cam_status status;
4695
4696         mtx_assert(&bus->eb_mtx, MA_OWNED);
4697         /* Make space for us in the device queue on our bus */
4698         devq = bus->sim->devq;
4699         mtx_lock(&devq->send_mtx);
4700         status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4701         mtx_unlock(&devq->send_mtx);
4702         if (status != CAM_REQ_CMP)
4703                 return (NULL);
4704
4705         device = (struct cam_ed *)malloc(sizeof(*device),
4706                                          M_CAMDEV, M_NOWAIT|M_ZERO);
4707         if (device == NULL)
4708                 return (NULL);
4709
4710         cam_init_pinfo(&device->devq_entry);
4711         device->target = target;
4712         device->lun_id = lun_id;
4713         device->sim = bus->sim;
4714         if (cam_ccbq_init(&device->ccbq,
4715                           bus->sim->max_dev_openings) != 0) {
4716                 free(device, M_CAMDEV);
4717                 return (NULL);
4718         }
4719         SLIST_INIT(&device->asyncs);
4720         SLIST_INIT(&device->periphs);
4721         device->generation = 0;
4722         device->flags = CAM_DEV_UNCONFIGURED;
4723         device->tag_delay_count = 0;
4724         device->tag_saved_openings = 0;
4725         device->refcount = 1;
4726         mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4727         callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4728         TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4729         /*
4730          * Hold a reference to our parent bus so it
4731          * will not go away before we do.
4732          */
4733         target->refcount++;
4734
4735         cur_device = TAILQ_FIRST(&target->ed_entries);
4736         while (cur_device != NULL && cur_device->lun_id < lun_id)
4737                 cur_device = TAILQ_NEXT(cur_device, links);
4738         if (cur_device != NULL)
4739                 TAILQ_INSERT_BEFORE(cur_device, device, links);
4740         else
4741                 TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4742         target->generation++;
4743         return (device);
4744 }
4745
4746 void
4747 xpt_acquire_device(struct cam_ed *device)
4748 {
4749         struct cam_eb *bus = device->target->bus;
4750
4751         mtx_lock(&bus->eb_mtx);
4752         device->refcount++;
4753         mtx_unlock(&bus->eb_mtx);
4754 }
4755
4756 void
4757 xpt_release_device(struct cam_ed *device)
4758 {
4759         struct cam_eb *bus = device->target->bus;
4760         struct cam_devq *devq;
4761
4762         mtx_lock(&bus->eb_mtx);
4763         if (--device->refcount > 0) {
4764                 mtx_unlock(&bus->eb_mtx);
4765                 return;
4766         }
4767
4768         TAILQ_REMOVE(&device->target->ed_entries, device,links);
4769         device->target->generation++;
4770         mtx_unlock(&bus->eb_mtx);
4771
4772         /* Release our slot in the devq */
4773         devq = bus->sim->devq;
4774         mtx_lock(&devq->send_mtx);
4775         cam_devq_resize(devq, devq->send_queue.array_size - 1);
4776         mtx_unlock(&devq->send_mtx);
4777
4778         KASSERT(SLIST_EMPTY(&device->periphs),
4779             ("destroying device, but periphs list is not empty"));
4780         KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4781             ("destroying device while still queued for ccbs"));
4782
4783         if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4784                 callout_stop(&device->callout);
4785
4786         xpt_release_target(device->target);
4787
4788         cam_ccbq_fini(&device->ccbq);
4789         /*
4790          * Free allocated memory.  free(9) does nothing if the
4791          * supplied pointer is NULL, so it is safe to call without
4792          * checking.
4793          */
4794         free(device->supported_vpds, M_CAMXPT);
4795         free(device->device_id, M_CAMXPT);
4796         free(device->ext_inq, M_CAMXPT);
4797         free(device->physpath, M_CAMXPT);
4798         free(device->rcap_buf, M_CAMXPT);
4799         free(device->serial_num, M_CAMXPT);
4800         taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4801 }
4802
4803 u_int32_t
4804 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4805 {
4806         int     result;
4807         struct  cam_ed *dev;
4808
4809         dev = path->device;
4810         mtx_lock(&dev->sim->devq->send_mtx);
4811         result = cam_ccbq_resize(&dev->ccbq, newopenings);
4812         mtx_unlock(&dev->sim->devq->send_mtx);
4813         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4814          || (dev->inq_flags & SID_CmdQue) != 0)
4815                 dev->tag_saved_openings = newopenings;
4816         return (result);
4817 }
4818
4819 static struct cam_eb *
4820 xpt_find_bus(path_id_t path_id)
4821 {
4822         struct cam_eb *bus;
4823
4824         xpt_lock_buses();
4825         for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4826              bus != NULL;
4827              bus = TAILQ_NEXT(bus, links)) {
4828                 if (bus->path_id == path_id) {
4829                         bus->refcount++;
4830                         break;
4831                 }
4832         }
4833         xpt_unlock_buses();
4834         return (bus);
4835 }
4836
4837 static struct cam_et *
4838 xpt_find_target(struct cam_eb *bus, target_id_t target_id)
4839 {
4840         struct cam_et *target;
4841
4842         mtx_assert(&bus->eb_mtx, MA_OWNED);
4843         for (target = TAILQ_FIRST(&bus->et_entries);
4844              target != NULL;
4845              target = TAILQ_NEXT(target, links)) {
4846                 if (target->target_id == target_id) {
4847                         target->refcount++;
4848                         break;
4849                 }
4850         }
4851         return (target);
4852 }
4853
4854 static struct cam_ed *
4855 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4856 {
4857         struct cam_ed *device;
4858
4859         mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4860         for (device = TAILQ_FIRST(&target->ed_entries);
4861              device != NULL;
4862              device = TAILQ_NEXT(device, links)) {
4863                 if (device->lun_id == lun_id) {
4864                         device->refcount++;
4865                         break;
4866                 }
4867         }
4868         return (device);
4869 }
4870
4871 void
4872 xpt_start_tags(struct cam_path *path)
4873 {
4874         struct ccb_relsim crs;
4875         struct cam_ed *device;
4876         struct cam_sim *sim;
4877         int    newopenings;
4878
4879         device = path->device;
4880         sim = path->bus->sim;
4881         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4882         xpt_freeze_devq(path, /*count*/1);
4883         device->inq_flags |= SID_CmdQue;
4884         if (device->tag_saved_openings != 0)
4885                 newopenings = device->tag_saved_openings;
4886         else
4887                 newopenings = min(device->maxtags,
4888                                   sim->max_tagged_dev_openings);
4889         xpt_dev_ccbq_resize(path, newopenings);
4890         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4891         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4892         crs.ccb_h.func_code = XPT_REL_SIMQ;
4893         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4894         crs.openings
4895             = crs.release_timeout
4896             = crs.qfrozen_cnt
4897             = 0;
4898         xpt_action((union ccb *)&crs);
4899 }
4900
4901 void
4902 xpt_stop_tags(struct cam_path *path)
4903 {
4904         struct ccb_relsim crs;
4905         struct cam_ed *device;
4906         struct cam_sim *sim;
4907
4908         device = path->device;
4909         sim = path->bus->sim;
4910         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4911         device->tag_delay_count = 0;
4912         xpt_freeze_devq(path, /*count*/1);
4913         device->inq_flags &= ~SID_CmdQue;
4914         xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4915         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4916         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4917         crs.ccb_h.func_code = XPT_REL_SIMQ;
4918         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4919         crs.openings
4920             = crs.release_timeout
4921             = crs.qfrozen_cnt
4922             = 0;
4923         xpt_action((union ccb *)&crs);
4924 }
4925
4926 static void
4927 xpt_boot_delay(void *arg)
4928 {
4929
4930         xpt_release_boot();
4931 }
4932
4933 static void
4934 xpt_config(void *arg)
4935 {
4936         /*
4937          * Now that interrupts are enabled, go find our devices
4938          */
4939         if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4940                 printf("xpt_config: failed to create taskqueue thread.\n");
4941
4942         /* Setup debugging path */
4943         if (cam_dflags != CAM_DEBUG_NONE) {
4944                 if (xpt_create_path(&cam_dpath, NULL,
4945                                     CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4946                                     CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4947                         printf("xpt_config: xpt_create_path() failed for debug"
4948                                " target %d:%d:%d, debugging disabled\n",
4949                                CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4950                         cam_dflags = CAM_DEBUG_NONE;
4951                 }
4952         } else
4953                 cam_dpath = NULL;
4954
4955         periphdriver_init(1);
4956         xpt_hold_boot();
4957         callout_init(&xsoftc.boot_callout, 1);
4958         callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
4959             xpt_boot_delay, NULL, 0);
4960         /* Fire up rescan thread. */
4961         if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4962             "cam", "scanner")) {
4963                 printf("xpt_config: failed to create rescan thread.\n");
4964         }
4965 }
4966
4967 void
4968 xpt_hold_boot(void)
4969 {
4970         xpt_lock_buses();
4971         xsoftc.buses_to_config++;
4972         xpt_unlock_buses();
4973 }
4974
4975 void
4976 xpt_release_boot(void)
4977 {
4978         xpt_lock_buses();
4979         xsoftc.buses_to_config--;
4980         if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4981                 struct  xpt_task *task;
4982
4983                 xsoftc.buses_config_done = 1;
4984                 xpt_unlock_buses();
4985                 /* Call manually because we don't have any busses */
4986                 task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
4987                 if (task != NULL) {
4988                         TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
4989                         taskqueue_enqueue(taskqueue_thread, &task->task);
4990                 }
4991         } else
4992                 xpt_unlock_buses();
4993 }
4994
4995 /*
4996  * If the given device only has one peripheral attached to it, and if that
4997  * peripheral is the passthrough driver, announce it.  This insures that the
4998  * user sees some sort of announcement for every peripheral in their system.
4999  */
5000 static int
5001 xptpassannouncefunc(struct cam_ed *device, void *arg)
5002 {
5003         struct cam_periph *periph;
5004         int i;
5005
5006         for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5007              periph = SLIST_NEXT(periph, periph_links), i++);
5008
5009         periph = SLIST_FIRST(&device->periphs);
5010         if ((i == 1)
5011          && (strncmp(periph->periph_name, "pass", 4) == 0))
5012                 xpt_announce_periph(periph, NULL);
5013
5014         return(1);
5015 }
5016
5017 static void
5018 xpt_finishconfig_task(void *context, int pending)
5019 {
5020
5021         periphdriver_init(2);
5022         /*
5023          * Check for devices with no "standard" peripheral driver
5024          * attached.  For any devices like that, announce the
5025          * passthrough driver so the user will see something.
5026          */
5027         if (!bootverbose)
5028                 xpt_for_all_devices(xptpassannouncefunc, NULL);
5029
5030         /* Release our hook so that the boot can continue. */
5031         config_intrhook_disestablish(xsoftc.xpt_config_hook);
5032         free(xsoftc.xpt_config_hook, M_CAMXPT);
5033         xsoftc.xpt_config_hook = NULL;
5034
5035         free(context, M_CAMXPT);
5036 }
5037
5038 cam_status
5039 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5040                    struct cam_path *path)
5041 {
5042         struct ccb_setasync csa;
5043         cam_status status;
5044         int xptpath = 0;
5045
5046         if (path == NULL) {
5047                 status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5048                                          CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5049                 if (status != CAM_REQ_CMP)
5050                         return (status);
5051                 xpt_path_lock(path);
5052                 xptpath = 1;
5053         }
5054
5055         xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5056         csa.ccb_h.func_code = XPT_SASYNC_CB;
5057         csa.event_enable = event;
5058         csa.callback = cbfunc;
5059         csa.callback_arg = cbarg;
5060         xpt_action((union ccb *)&csa);
5061         status = csa.ccb_h.status;
5062
5063         if (xptpath) {
5064                 xpt_path_unlock(path);
5065                 xpt_free_path(path);
5066         }
5067
5068         if ((status == CAM_REQ_CMP) &&
5069             (csa.event_enable & AC_FOUND_DEVICE)) {
5070                 /*
5071                  * Get this peripheral up to date with all
5072                  * the currently existing devices.
5073                  */
5074                 xpt_for_all_devices(xptsetasyncfunc, &csa);
5075         }
5076         if ((status == CAM_REQ_CMP) &&
5077             (csa.event_enable & AC_PATH_REGISTERED)) {
5078                 /*
5079                  * Get this peripheral up to date with all
5080                  * the currently existing busses.
5081                  */
5082                 xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5083         }
5084
5085         return (status);
5086 }
5087
5088 static void
5089 xptaction(struct cam_sim *sim, union ccb *work_ccb)
5090 {
5091         CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5092
5093         switch (work_ccb->ccb_h.func_code) {
5094         /* Common cases first */
5095         case XPT_PATH_INQ:              /* Path routing inquiry */
5096         {
5097                 struct ccb_pathinq *cpi;
5098
5099                 cpi = &work_ccb->cpi;
5100                 cpi->version_num = 1; /* XXX??? */
5101                 cpi->hba_inquiry = 0;
5102                 cpi->target_sprt = 0;
5103                 cpi->hba_misc = 0;
5104                 cpi->hba_eng_cnt = 0;
5105                 cpi->max_target = 0;
5106                 cpi->max_lun = 0;
5107                 cpi->initiator_id = 0;
5108                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5109                 strncpy(cpi->hba_vid, "", HBA_IDLEN);
5110                 strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5111                 cpi->unit_number = sim->unit_number;
5112                 cpi->bus_id = sim->bus_id;
5113                 cpi->base_transfer_speed = 0;
5114                 cpi->protocol = PROTO_UNSPECIFIED;
5115                 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5116                 cpi->transport = XPORT_UNSPECIFIED;
5117                 cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5118                 cpi->ccb_h.status = CAM_REQ_CMP;
5119                 xpt_done(work_ccb);
5120                 break;
5121         }
5122         default:
5123                 work_ccb->ccb_h.status = CAM_REQ_INVALID;
5124                 xpt_done(work_ccb);
5125                 break;
5126         }
5127 }
5128
5129 /*
5130  * The xpt as a "controller" has no interrupt sources, so polling
5131  * is a no-op.
5132  */
5133 static void
5134 xptpoll(struct cam_sim *sim)
5135 {
5136 }
5137
5138 void
5139 xpt_lock_buses(void)
5140 {
5141         mtx_lock(&xsoftc.xpt_topo_lock);
5142 }
5143
5144 void
5145 xpt_unlock_buses(void)
5146 {
5147         mtx_unlock(&xsoftc.xpt_topo_lock);
5148 }
5149
5150 struct mtx *
5151 xpt_path_mtx(struct cam_path *path)
5152 {
5153
5154         return (&path->device->device_mtx);
5155 }
5156
5157 static void
5158 xpt_done_process(struct ccb_hdr *ccb_h)
5159 {
5160         struct cam_sim *sim;
5161         struct cam_devq *devq;
5162         struct mtx *mtx = NULL;
5163
5164         if (ccb_h->flags & CAM_HIGH_POWER) {
5165                 struct highpowerlist    *hphead;
5166                 struct cam_ed           *device;
5167
5168                 mtx_lock(&xsoftc.xpt_highpower_lock);
5169                 hphead = &xsoftc.highpowerq;
5170
5171                 device = STAILQ_FIRST(hphead);
5172
5173                 /*
5174                  * Increment the count since this command is done.
5175                  */
5176                 xsoftc.num_highpower++;
5177
5178                 /*
5179                  * Any high powered commands queued up?
5180                  */
5181                 if (device != NULL) {
5182
5183                         STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5184                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5185
5186                         mtx_lock(&device->sim->devq->send_mtx);
5187                         xpt_release_devq_device(device,
5188                                          /*count*/1, /*runqueue*/TRUE);
5189                         mtx_unlock(&device->sim->devq->send_mtx);
5190                 } else
5191                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5192         }
5193
5194         sim = ccb_h->path->bus->sim;
5195
5196         if (ccb_h->status & CAM_RELEASE_SIMQ) {
5197                 xpt_release_simq(sim, /*run_queue*/FALSE);
5198                 ccb_h->status &= ~CAM_RELEASE_SIMQ;
5199         }
5200
5201         if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5202          && (ccb_h->status & CAM_DEV_QFRZN)) {
5203                 xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5204                 ccb_h->status &= ~CAM_DEV_QFRZN;
5205         }
5206
5207         devq = sim->devq;
5208         if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5209                 struct cam_ed *dev = ccb_h->path->device;
5210
5211                 mtx_lock(&devq->send_mtx);
5212                 devq->send_active--;
5213                 devq->send_openings++;
5214                 cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5215
5216                 if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5217                   && (dev->ccbq.dev_active == 0))) {
5218                         dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5219                         xpt_release_devq_device(dev, /*count*/1,
5220                                          /*run_queue*/FALSE);
5221                 }
5222
5223                 if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5224                   && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5225                         dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5226                         xpt_release_devq_device(dev, /*count*/1,
5227                                          /*run_queue*/FALSE);
5228                 }
5229
5230                 if (!device_is_queued(dev))
5231                         (void)xpt_schedule_devq(devq, dev);
5232                 xpt_run_devq(devq);
5233                 mtx_unlock(&devq->send_mtx);
5234
5235                 if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5236                         mtx = xpt_path_mtx(ccb_h->path);
5237                         mtx_lock(mtx);
5238
5239                         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5240                          && (--dev->tag_delay_count == 0))
5241                                 xpt_start_tags(ccb_h->path);
5242                 }
5243         }
5244
5245         if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5246                 if (mtx == NULL) {
5247                         mtx = xpt_path_mtx(ccb_h->path);
5248                         mtx_lock(mtx);
5249                 }
5250         } else {
5251                 if (mtx != NULL) {
5252                         mtx_unlock(mtx);
5253                         mtx = NULL;
5254                 }
5255         }
5256
5257         /* Call the peripheral driver's callback */
5258         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5259         (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5260         if (mtx != NULL)
5261                 mtx_unlock(mtx);
5262 }
5263
5264 void
5265 xpt_done_td(void *arg)
5266 {
5267         struct cam_doneq *queue = arg;
5268         struct ccb_hdr *ccb_h;
5269         STAILQ_HEAD(, ccb_hdr)  doneq;
5270
5271         STAILQ_INIT(&doneq);
5272         mtx_lock(&queue->cam_doneq_mtx);
5273         while (1) {
5274                 while (STAILQ_EMPTY(&queue->cam_doneq)) {
5275                         queue->cam_doneq_sleep = 1;
5276                         msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5277                             PRIBIO, "-", 0);
5278                         queue->cam_doneq_sleep = 0;
5279                 }
5280                 STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5281                 mtx_unlock(&queue->cam_doneq_mtx);
5282
5283                 THREAD_NO_SLEEPING();
5284                 while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5285                         STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5286                         xpt_done_process(ccb_h);
5287                 }
5288                 THREAD_SLEEPING_OK();
5289
5290                 mtx_lock(&queue->cam_doneq_mtx);
5291         }
5292 }
5293
5294 static void
5295 camisr_runqueue(void)
5296 {
5297         struct  ccb_hdr *ccb_h;
5298         struct cam_doneq *queue;
5299         int i;
5300
5301         /* Process global queues. */
5302         for (i = 0; i < cam_num_doneqs; i++) {
5303                 queue = &cam_doneqs[i];
5304                 mtx_lock(&queue->cam_doneq_mtx);
5305                 while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5306                         STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5307                         mtx_unlock(&queue->cam_doneq_mtx);
5308                         xpt_done_process(ccb_h);
5309                         mtx_lock(&queue->cam_doneq_mtx);
5310                 }
5311                 mtx_unlock(&queue->cam_doneq_mtx);
5312         }
5313 }