sys/cam/cam_xpt.c

   1 /*-
   2  * Implementation of the Common Access Method Transport (XPT) layer.
   3  *
   4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
   5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification, immediately at the beginning of the file.
  14  * 2. The name of the author may not be used to endorse or promote products
  15  *    derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/bus.h>
  35 #include <sys/systm.h>
  36 #include <sys/types.h>
  37 #include <sys/malloc.h>
  38 #include <sys/kernel.h>
  39 #include <sys/time.h>
  40 #include <sys/conf.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/interrupt.h>
  43 #include <sys/proc.h>
  44 #include <sys/sbuf.h>
  45 #include <sys/smp.h>
  46 #include <sys/taskqueue.h>
  47
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/sysctl.h>
  51 #include <sys/kthread.h>
  52
  53 #include <cam/cam.h>
  54 #include <cam/cam_ccb.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/cam_queue.h>
  57 #include <cam/cam_sim.h>
  58 #include <cam/cam_xpt.h>
  59 #include <cam/cam_xpt_sim.h>
  60 #include <cam/cam_xpt_periph.h>
  61 #include <cam/cam_xpt_internal.h>
  62 #include <cam/cam_debug.h>
  63 #include <cam/cam_compat.h>
  64
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_message.h>
  67 #include <cam/scsi/scsi_pass.h>
  68
  69 #include <machine/md_var.h>     /* geometry translation */
  70 #include <machine/stdarg.h>     /* for xpt_print below */
  71
  72 #include "opt_cam.h"
  73
  74 /*
  75  * This is the maximum number of high powered commands (e.g. start unit)
  76  * that can be outstanding at a particular time.
  77  */
  78 #ifndef CAM_MAX_HIGHPOWER
  79 #define CAM_MAX_HIGHPOWER  4
  80 #endif
  81
  82 /* Datastructures internal to the xpt layer */
  83 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
  84 MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
  85 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
  86 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
  87
  88 /* Object for defering XPT actions to a taskqueue */
  89 struct xpt_task {
  90         struct task     task;
  91         void            *data1;
  92         uintptr_t       data2;
  93 };
  94
  95 typedef enum {
  96         XPT_FLAG_OPEN           = 0x01
  97 } xpt_flags;
  98
  99 struct xpt_softc {
 100         xpt_flags               flags;
 101
 102         /* number of high powered commands that can go through right now */
 103         STAILQ_HEAD(highpowerlist, cam_ed)      highpowerq;
 104         int                     num_highpower;
 105
 106         /* queue for handling async rescan requests. */
 107         TAILQ_HEAD(, ccb_hdr) ccb_scanq;
 108         int buses_to_config;
 109         int buses_config_done;
 110
 111         /* Registered busses */
 112         TAILQ_HEAD(,cam_eb)     xpt_busses;
 113         u_int                   bus_generation;
 114
 115         struct intr_config_hook *xpt_config_hook;
 116
 117         int                     boot_delay;
 118         struct callout          boot_callout;
 119
 120         struct mtx              xpt_topo_lock;
 121         struct mtx              xpt_lock;
 122         struct taskqueue        *xpt_taskq;
 123 };
 124
 125 typedef enum {
 126         DM_RET_COPY             = 0x01,
 127         DM_RET_FLAG_MASK        = 0x0f,
 128         DM_RET_NONE             = 0x00,
 129         DM_RET_STOP             = 0x10,
 130         DM_RET_DESCEND          = 0x20,
 131         DM_RET_ERROR            = 0x30,
 132         DM_RET_ACTION_MASK      = 0xf0
 133 } dev_match_ret;
 134
 135 typedef enum {
 136         XPT_DEPTH_BUS,
 137         XPT_DEPTH_TARGET,
 138         XPT_DEPTH_DEVICE,
 139         XPT_DEPTH_PERIPH
 140 } xpt_traverse_depth;
 141
 142 struct xpt_traverse_config {
 143         xpt_traverse_depth      depth;
 144         void                    *tr_func;
 145         void                    *tr_arg;
 146 };
 147
 148 typedef int     xpt_busfunc_t (struct cam_eb *bus, void *arg);
 149 typedef int     xpt_targetfunc_t (struct cam_et *target, void *arg);
 150 typedef int     xpt_devicefunc_t (struct cam_ed *device, void *arg);
 151 typedef int     xpt_periphfunc_t (struct cam_periph *periph, void *arg);
 152 typedef int     xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
 153
 154 /* Transport layer configuration information */
 155 static struct xpt_softc xsoftc;
 156
 157 TUNABLE_INT("kern.cam.boot_delay", &xsoftc.boot_delay);
 158 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
 159            &xsoftc.boot_delay, 0, "Bus registration wait time");
 160
 161 struct cam_doneq {
 162         struct mtx_padalign     cam_doneq_mtx;
 163         STAILQ_HEAD(, ccb_hdr)  cam_doneq;
 164         int                     cam_doneq_sleep;
 165 };
 166
 167 static struct cam_doneq cam_doneqs[MAXCPU];
 168 static int cam_num_doneqs;
 169 static struct proc *cam_proc;
 170
 171 TUNABLE_INT("kern.cam.num_doneqs", &cam_num_doneqs);
 172 SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
 173            &cam_num_doneqs, 0, "Number of completion queues/threads");
 174
 175 struct cam_periph *xpt_periph;
 176
 177 static periph_init_t xpt_periph_init;
 178
 179 static struct periph_driver xpt_driver =
 180 {
 181         xpt_periph_init, "xpt",
 182         TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
 183         CAM_PERIPH_DRV_EARLY
 184 };
 185
 186 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
 187
 188 static d_open_t xptopen;
 189 static d_close_t xptclose;
 190 static d_ioctl_t xptioctl;
 191 static d_ioctl_t xptdoioctl;
 192
 193 static struct cdevsw xpt_cdevsw = {
 194         .d_version =    D_VERSION,
 195         .d_flags =      0,
 196         .d_open =       xptopen,
 197         .d_close =      xptclose,
 198         .d_ioctl =      xptioctl,
 199         .d_name =       "xpt",
 200 };
 201
 202 /* Storage for debugging datastructures */
 203 struct cam_path *cam_dpath;
 204 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
 205 TUNABLE_INT("kern.cam.dflags", &cam_dflags);
 206 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RW,
 207         &cam_dflags, 0, "Enabled debug flags");
 208 u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
 209 TUNABLE_INT("kern.cam.debug_delay", &cam_debug_delay);
 210 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RW,
 211         &cam_debug_delay, 0, "Delay in us after each debug message");
 212
 213 /* Our boot-time initialization hook */
 214 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 215
 216 static moduledata_t cam_moduledata = {
 217         "cam",
 218         cam_module_event_handler,
 219         NULL
 220 };
 221
 222 static int      xpt_init(void *);
 223
 224 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
 225 MODULE_VERSION(cam, 1);
 226
 227
 228 static void             xpt_async_bcast(struct async_list *async_head,
 229                                         u_int32_t async_code,
 230                                         struct cam_path *path,
 231                                         void *async_arg);
 232 static path_id_t xptnextfreepathid(void);
 233 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
 234 static union ccb *xpt_get_ccb(struct cam_periph *periph);
 235 static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
 236 static void      xpt_run_allocq(struct cam_periph *periph, int sleep);
 237 static void      xpt_run_allocq_task(void *context, int pending);
 238 static void      xpt_run_devq(struct cam_devq *devq);
 239 static timeout_t xpt_release_devq_timeout;
 240 static void      xpt_release_simq_timeout(void *arg) __unused;
 241 static void      xpt_acquire_bus(struct cam_eb *bus);
 242 static void      xpt_release_bus(struct cam_eb *bus);
 243 static int       xpt_release_devq_device(struct cam_ed *dev, u_int count,
 244                     int run_queue);
 245 static struct cam_et*
 246                  xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
 247 static void      xpt_acquire_target(struct cam_et *target);
 248 static void      xpt_release_target(struct cam_et *target);
 249 static struct cam_eb*
 250                  xpt_find_bus(path_id_t path_id);
 251 static struct cam_et*
 252                  xpt_find_target(struct cam_eb *bus, target_id_t target_id);
 253 static struct cam_ed*
 254                  xpt_find_device(struct cam_et *target, lun_id_t lun_id);
 255 static void      xpt_config(void *arg);
 256 static int       xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 257                                  u_int32_t new_priority);
 258 static xpt_devicefunc_t xptpassannouncefunc;
 259 static void      xptaction(struct cam_sim *sim, union ccb *work_ccb);
 260 static void      xptpoll(struct cam_sim *sim);
 261 static void      camisr_runqueue(void);
 262 static void      xpt_done_process(struct ccb_hdr *ccb_h);
 263 static void      xpt_done_td(void *);
 264 static dev_match_ret    xptbusmatch(struct dev_match_pattern *patterns,
 265                                     u_int num_patterns, struct cam_eb *bus);
 266 static dev_match_ret    xptdevicematch(struct dev_match_pattern *patterns,
 267                                        u_int num_patterns,
 268                                        struct cam_ed *device);
 269 static dev_match_ret    xptperiphmatch(struct dev_match_pattern *patterns,
 270                                        u_int num_patterns,
 271                                        struct cam_periph *periph);
 272 static xpt_busfunc_t    xptedtbusfunc;
 273 static xpt_targetfunc_t xptedttargetfunc;
 274 static xpt_devicefunc_t xptedtdevicefunc;
 275 static xpt_periphfunc_t xptedtperiphfunc;
 276 static xpt_pdrvfunc_t   xptplistpdrvfunc;
 277 static xpt_periphfunc_t xptplistperiphfunc;
 278 static int              xptedtmatch(struct ccb_dev_match *cdm);
 279 static int              xptperiphlistmatch(struct ccb_dev_match *cdm);
 280 static int              xptbustraverse(struct cam_eb *start_bus,
 281                                        xpt_busfunc_t *tr_func, void *arg);
 282 static int              xpttargettraverse(struct cam_eb *bus,
 283                                           struct cam_et *start_target,
 284                                           xpt_targetfunc_t *tr_func, void *arg);
 285 static int              xptdevicetraverse(struct cam_et *target,
 286                                           struct cam_ed *start_device,
 287                                           xpt_devicefunc_t *tr_func, void *arg);
 288 static int              xptperiphtraverse(struct cam_ed *device,
 289                                           struct cam_periph *start_periph,
 290                                           xpt_periphfunc_t *tr_func, void *arg);
 291 static int              xptpdrvtraverse(struct periph_driver **start_pdrv,
 292                                         xpt_pdrvfunc_t *tr_func, void *arg);
 293 static int              xptpdperiphtraverse(struct periph_driver **pdrv,
 294                                             struct cam_periph *start_periph,
 295                                             xpt_periphfunc_t *tr_func,
 296                                             void *arg);
 297 static xpt_busfunc_t    xptdefbusfunc;
 298 static xpt_targetfunc_t xptdeftargetfunc;
 299 static xpt_devicefunc_t xptdefdevicefunc;
 300 static xpt_periphfunc_t xptdefperiphfunc;
 301 static void             xpt_finishconfig_task(void *context, int pending);
 302 static void             xpt_dev_async_default(u_int32_t async_code,
 303                                               struct cam_eb *bus,
 304                                               struct cam_et *target,
 305                                               struct cam_ed *device,
 306                                               void *async_arg);
 307 static struct cam_ed *  xpt_alloc_device_default(struct cam_eb *bus,
 308                                                  struct cam_et *target,
 309                                                  lun_id_t lun_id);
 310 static xpt_devicefunc_t xptsetasyncfunc;
 311 static xpt_busfunc_t    xptsetasyncbusfunc;
 312 static cam_status       xptregister(struct cam_periph *periph,
 313                                     void *arg);
 314 static __inline int device_is_queued(struct cam_ed *device);
 315
 316 static __inline int
 317 xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
 318 {
 319         int     retval;
 320
 321         mtx_assert(&devq->send_mtx, MA_OWNED);
 322         if ((dev->ccbq.queue.entries > 0) &&
 323             (dev->ccbq.dev_openings > 0) &&
 324             (dev->ccbq.queue.qfrozen_cnt == 0)) {
 325                 /*
 326                  * The priority of a device waiting for controller
 327                  * resources is that of the highest priority CCB
 328                  * enqueued.
 329                  */
 330                 retval =
 331                     xpt_schedule_dev(&devq->send_queue,
 332                                      &dev->devq_entry,
 333                                      CAMQ_GET_PRIO(&dev->ccbq.queue));
 334         } else {
 335                 retval = 0;
 336         }
 337         return (retval);
 338 }
 339
 340 static __inline int
 341 device_is_queued(struct cam_ed *device)
 342 {
 343         return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
 344 }
 345
 346 static void
 347 xpt_periph_init()
 348 {
 349         make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
 350 }
 351
 352 static int
 353 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 354 {
 355
 356         /*
 357          * Only allow read-write access.
 358          */
 359         if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
 360                 return(EPERM);
 361
 362         /*
 363          * We don't allow nonblocking access.
 364          */
 365         if ((flags & O_NONBLOCK) != 0) {
 366                 printf("%s: can't do nonblocking access\n", devtoname(dev));
 367                 return(ENODEV);
 368         }
 369
 370         /* Mark ourselves open */
 371         mtx_lock(&xsoftc.xpt_lock);
 372         xsoftc.flags |= XPT_FLAG_OPEN;
 373         mtx_unlock(&xsoftc.xpt_lock);
 374
 375         return(0);
 376 }
 377
 378 static int
 379 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 380 {
 381
 382         /* Mark ourselves closed */
 383         mtx_lock(&xsoftc.xpt_lock);
 384         xsoftc.flags &= ~XPT_FLAG_OPEN;
 385         mtx_unlock(&xsoftc.xpt_lock);
 386
 387         return(0);
 388 }
 389
 390 /*
 391  * Don't automatically grab the xpt softc lock here even though this is going
 392  * through the xpt device.  The xpt device is really just a back door for
 393  * accessing other devices and SIMs, so the right thing to do is to grab
 394  * the appropriate SIM lock once the bus/SIM is located.
 395  */
 396 static int
 397 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 398 {
 399         int error;
 400
 401         if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
 402                 error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
 403         }
 404         return (error);
 405 }
 406
 407 static int
 408 xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 409 {
 410         int error;
 411
 412         error = 0;
 413
 414         switch(cmd) {
 415         /*
 416          * For the transport layer CAMIOCOMMAND ioctl, we really only want
 417          * to accept CCB types that don't quite make sense to send through a
 418          * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
 419          * in the CAM spec.
 420          */
 421         case CAMIOCOMMAND: {
 422                 union ccb *ccb;
 423                 union ccb *inccb;
 424                 struct cam_eb *bus;
 425
 426                 inccb = (union ccb *)addr;
 427
 428                 bus = xpt_find_bus(inccb->ccb_h.path_id);
 429                 if (bus == NULL)
 430                         return (EINVAL);
 431
 432                 switch (inccb->ccb_h.func_code) {
 433                 case XPT_SCAN_BUS:
 434                 case XPT_RESET_BUS:
 435                         if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
 436                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 437                                 xpt_release_bus(bus);
 438                                 return (EINVAL);
 439                         }
 440                         break;
 441                 case XPT_SCAN_TGT:
 442                         if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
 443                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 444                                 xpt_release_bus(bus);
 445                                 return (EINVAL);
 446                         }
 447                         break;
 448                 default:
 449                         break;
 450                 }
 451
 452                 switch(inccb->ccb_h.func_code) {
 453                 case XPT_SCAN_BUS:
 454                 case XPT_RESET_BUS:
 455                 case XPT_PATH_INQ:
 456                 case XPT_ENG_INQ:
 457                 case XPT_SCAN_LUN:
 458                 case XPT_SCAN_TGT:
 459
 460                         ccb = xpt_alloc_ccb();
 461
 462                         /*
 463                          * Create a path using the bus, target, and lun the
 464                          * user passed in.
 465                          */
 466                         if (xpt_create_path(&ccb->ccb_h.path, NULL,
 467                                             inccb->ccb_h.path_id,
 468                                             inccb->ccb_h.target_id,
 469                                             inccb->ccb_h.target_lun) !=
 470                                             CAM_REQ_CMP){
 471                                 error = EINVAL;
 472                                 xpt_free_ccb(ccb);
 473                                 break;
 474                         }
 475                         /* Ensure all of our fields are correct */
 476                         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
 477                                       inccb->ccb_h.pinfo.priority);
 478                         xpt_merge_ccb(ccb, inccb);
 479                         xpt_path_lock(ccb->ccb_h.path);
 480                         cam_periph_runccb(ccb, NULL, 0, 0, NULL);
 481                         xpt_path_unlock(ccb->ccb_h.path);
 482                         bcopy(ccb, inccb, sizeof(union ccb));
 483                         xpt_free_path(ccb->ccb_h.path);
 484                         xpt_free_ccb(ccb);
 485                         break;
 486
 487                 case XPT_DEBUG: {
 488                         union ccb ccb;
 489
 490                         /*
 491                          * This is an immediate CCB, so it's okay to
 492                          * allocate it on the stack.
 493                          */
 494
 495                         /*
 496                          * Create a path using the bus, target, and lun the
 497                          * user passed in.
 498                          */
 499                         if (xpt_create_path(&ccb.ccb_h.path, NULL,
 500                                             inccb->ccb_h.path_id,
 501                                             inccb->ccb_h.target_id,
 502                                             inccb->ccb_h.target_lun) !=
 503                                             CAM_REQ_CMP){
 504                                 error = EINVAL;
 505                                 break;
 506                         }
 507                         /* Ensure all of our fields are correct */
 508                         xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
 509                                       inccb->ccb_h.pinfo.priority);
 510                         xpt_merge_ccb(&ccb, inccb);
 511                         xpt_action(&ccb);
 512                         bcopy(&ccb, inccb, sizeof(union ccb));
 513                         xpt_free_path(ccb.ccb_h.path);
 514                         break;
 515
 516                 }
 517                 case XPT_DEV_MATCH: {
 518                         struct cam_periph_map_info mapinfo;
 519                         struct cam_path *old_path;
 520
 521                         /*
 522                          * We can't deal with physical addresses for this
 523                          * type of transaction.
 524                          */
 525                         if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
 526                             CAM_DATA_VADDR) {
 527                                 error = EINVAL;
 528                                 break;
 529                         }
 530
 531                         /*
 532                          * Save this in case the caller had it set to
 533                          * something in particular.
 534                          */
 535                         old_path = inccb->ccb_h.path;
 536
 537                         /*
 538                          * We really don't need a path for the matching
 539                          * code.  The path is needed because of the
 540                          * debugging statements in xpt_action().  They
 541                          * assume that the CCB has a valid path.
 542                          */
 543                         inccb->ccb_h.path = xpt_periph->path;
 544
 545                         bzero(&mapinfo, sizeof(mapinfo));
 546
 547                         /*
 548                          * Map the pattern and match buffers into kernel
 549                          * virtual address space.
 550                          */
 551                         error = cam_periph_mapmem(inccb, &mapinfo);
 552
 553                         if (error) {
 554                                 inccb->ccb_h.path = old_path;
 555                                 break;
 556                         }
 557
 558                         /*
 559                          * This is an immediate CCB, we can send it on directly.
 560                          */
 561                         xpt_action(inccb);
 562
 563                         /*
 564                          * Map the buffers back into user space.
 565                          */
 566                         cam_periph_unmapmem(inccb, &mapinfo);
 567
 568                         inccb->ccb_h.path = old_path;
 569
 570                         error = 0;
 571                         break;
 572                 }
 573                 default:
 574                         error = ENOTSUP;
 575                         break;
 576                 }
 577                 xpt_release_bus(bus);
 578                 break;
 579         }
 580         /*
 581          * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
 582          * with the periphal driver name and unit name filled in.  The other
 583          * fields don't really matter as input.  The passthrough driver name
 584          * ("pass"), and unit number are passed back in the ccb.  The current
 585          * device generation number, and the index into the device peripheral
 586          * driver list, and the status are also passed back.  Note that
 587          * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
 588          * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
 589          * (or rather should be) impossible for the device peripheral driver
 590          * list to change since we look at the whole thing in one pass, and
 591          * we do it with lock protection.
 592          *
 593          */
 594         case CAMGETPASSTHRU: {
 595                 union ccb *ccb;
 596                 struct cam_periph *periph;
 597                 struct periph_driver **p_drv;
 598                 char   *name;
 599                 u_int unit;
 600                 int base_periph_found;
 601
 602                 ccb = (union ccb *)addr;
 603                 unit = ccb->cgdl.unit_number;
 604                 name = ccb->cgdl.periph_name;
 605                 base_periph_found = 0;
 606
 607                 /*
 608                  * Sanity check -- make sure we don't get a null peripheral
 609                  * driver name.
 610                  */
 611                 if (*ccb->cgdl.periph_name == '\0') {
 612                         error = EINVAL;
 613                         break;
 614                 }
 615
 616                 /* Keep the list from changing while we traverse it */
 617                 xpt_lock_buses();
 618
 619                 /* first find our driver in the list of drivers */
 620                 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
 621                         if (strcmp((*p_drv)->driver_name, name) == 0)
 622                                 break;
 623
 624                 if (*p_drv == NULL) {
 625                         xpt_unlock_buses();
 626                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 627                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 628                         *ccb->cgdl.periph_name = '\0';
 629                         ccb->cgdl.unit_number = 0;
 630                         error = ENOENT;
 631                         break;
 632                 }
 633
 634                 /*
 635                  * Run through every peripheral instance of this driver
 636                  * and check to see whether it matches the unit passed
 637                  * in by the user.  If it does, get out of the loops and
 638                  * find the passthrough driver associated with that
 639                  * peripheral driver.
 640                  */
 641                 for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
 642                      periph = TAILQ_NEXT(periph, unit_links)) {
 643
 644                         if (periph->unit_number == unit)
 645                                 break;
 646                 }
 647                 /*
 648                  * If we found the peripheral driver that the user passed
 649                  * in, go through all of the peripheral drivers for that
 650                  * particular device and look for a passthrough driver.
 651                  */
 652                 if (periph != NULL) {
 653                         struct cam_ed *device;
 654                         int i;
 655
 656                         base_periph_found = 1;
 657                         device = periph->path->device;
 658                         for (i = 0, periph = SLIST_FIRST(&device->periphs);
 659                              periph != NULL;
 660                              periph = SLIST_NEXT(periph, periph_links), i++) {
 661                                 /*
 662                                  * Check to see whether we have a
 663                                  * passthrough device or not.
 664                                  */
 665                                 if (strcmp(periph->periph_name, "pass") == 0) {
 666                                         /*
 667                                          * Fill in the getdevlist fields.
 668                                          */
 669                                         strcpy(ccb->cgdl.periph_name,
 670                                                periph->periph_name);
 671                                         ccb->cgdl.unit_number =
 672                                                 periph->unit_number;
 673                                         if (SLIST_NEXT(periph, periph_links))
 674                                                 ccb->cgdl.status =
 675                                                         CAM_GDEVLIST_MORE_DEVS;
 676                                         else
 677                                                 ccb->cgdl.status =
 678                                                        CAM_GDEVLIST_LAST_DEVICE;
 679                                         ccb->cgdl.generation =
 680                                                 device->generation;
 681                                         ccb->cgdl.index = i;
 682                                         /*
 683                                          * Fill in some CCB header fields
 684                                          * that the user may want.
 685                                          */
 686                                         ccb->ccb_h.path_id =
 687                                                 periph->path->bus->path_id;
 688                                         ccb->ccb_h.target_id =
 689                                                 periph->path->target->target_id;
 690                                         ccb->ccb_h.target_lun =
 691                                                 periph->path->device->lun_id;
 692                                         ccb->ccb_h.status = CAM_REQ_CMP;
 693                                         break;
 694                                 }
 695                         }
 696                 }
 697
 698                 /*
 699                  * If the periph is null here, one of two things has
 700                  * happened.  The first possibility is that we couldn't
 701                  * find the unit number of the particular peripheral driver
 702                  * that the user is asking about.  e.g. the user asks for
 703                  * the passthrough driver for "da11".  We find the list of
 704                  * "da" peripherals all right, but there is no unit 11.
 705                  * The other possibility is that we went through the list
 706                  * of peripheral drivers attached to the device structure,
 707                  * but didn't find one with the name "pass".  Either way,
 708                  * we return ENOENT, since we couldn't find something.
 709                  */
 710                 if (periph == NULL) {
 711                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 712                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 713                         *ccb->cgdl.periph_name = '\0';
 714                         ccb->cgdl.unit_number = 0;
 715                         error = ENOENT;
 716                         /*
 717                          * It is unfortunate that this is even necessary,
 718                          * but there are many, many clueless users out there.
 719                          * If this is true, the user is looking for the
 720                          * passthrough driver, but doesn't have one in his
 721                          * kernel.
 722                          */
 723                         if (base_periph_found == 1) {
 724                                 printf("xptioctl: pass driver is not in the "
 725                                        "kernel\n");
 726                                 printf("xptioctl: put \"device pass\" in "
 727                                        "your kernel config file\n");
 728                         }
 729                 }
 730                 xpt_unlock_buses();
 731                 break;
 732                 }
 733         default:
 734                 error = ENOTTY;
 735                 break;
 736         }
 737
 738         return(error);
 739 }
 740
 741 static int
 742 cam_module_event_handler(module_t mod, int what, void *arg)
 743 {
 744         int error;
 745
 746         switch (what) {
 747         case MOD_LOAD:
 748                 if ((error = xpt_init(NULL)) != 0)
 749                         return (error);
 750                 break;
 751         case MOD_UNLOAD:
 752                 return EBUSY;
 753         default:
 754                 return EOPNOTSUPP;
 755         }
 756
 757         return 0;
 758 }
 759
 760 static void
 761 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
 762 {
 763
 764         if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
 765                 xpt_free_path(done_ccb->ccb_h.path);
 766                 xpt_free_ccb(done_ccb);
 767         } else {
 768                 done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
 769                 (*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
 770         }
 771         xpt_release_boot();
 772 }
 773
 774 /* thread to handle bus rescans */
 775 static void
 776 xpt_scanner_thread(void *dummy)
 777 {
 778         union ccb       *ccb;
 779         struct cam_path  path;
 780
 781         xpt_lock_buses();
 782         for (;;) {
 783                 if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
 784                         msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
 785                                "ccb_scanq", 0);
 786                 if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
 787                         TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 788                         xpt_unlock_buses();
 789
 790                         /*
 791                          * Since lock can be dropped inside and path freed
 792                          * by completion callback even before return here,
 793                          * take our own path copy for reference.
 794                          */
 795                         xpt_copy_path(&path, ccb->ccb_h.path);
 796                         xpt_path_lock(&path);
 797                         xpt_action(ccb);
 798                         xpt_path_unlock(&path);
 799                         xpt_release_path(&path);
 800
 801                         xpt_lock_buses();
 802                 }
 803         }
 804 }
 805
 806 void
 807 xpt_rescan(union ccb *ccb)
 808 {
 809         struct ccb_hdr *hdr;
 810
 811         /* Prepare request */
 812         if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
 813             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 814                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 815         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 816             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 817                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 818         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 819             ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
 820                 ccb->ccb_h.func_code = XPT_SCAN_LUN;
 821         else {
 822                 xpt_print(ccb->ccb_h.path, "illegal scan path\n");
 823                 xpt_free_path(ccb->ccb_h.path);
 824                 xpt_free_ccb(ccb);
 825                 return;
 826         }
 827         ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
 828         ccb->ccb_h.cbfcnp = xpt_rescan_done;
 829         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
 830         /* Don't make duplicate entries for the same paths. */
 831         xpt_lock_buses();
 832         if (ccb->ccb_h.ppriv_ptr1 == NULL) {
 833                 TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
 834                         if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
 835                                 wakeup(&xsoftc.ccb_scanq);
 836                                 xpt_unlock_buses();
 837                                 xpt_print(ccb->ccb_h.path, "rescan already queued\n");
 838                                 xpt_free_path(ccb->ccb_h.path);
 839                                 xpt_free_ccb(ccb);
 840                                 return;
 841                         }
 842                 }
 843         }
 844         TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 845         xsoftc.buses_to_config++;
 846         wakeup(&xsoftc.ccb_scanq);
 847         xpt_unlock_buses();
 848 }
 849
 850 /* Functions accessed by the peripheral drivers */
 851 static int
 852 xpt_init(void *dummy)
 853 {
 854         struct cam_sim *xpt_sim;
 855         struct cam_path *path;
 856         struct cam_devq *devq;
 857         cam_status status;
 858         int error, i;
 859
 860         TAILQ_INIT(&xsoftc.xpt_busses);
 861         TAILQ_INIT(&xsoftc.ccb_scanq);
 862         STAILQ_INIT(&xsoftc.highpowerq);
 863         xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
 864
 865         mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
 866         mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF);
 867         xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
 868             taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
 869
 870 #ifdef CAM_BOOT_DELAY
 871         /*
 872          * Override this value at compile time to assist our users
 873          * who don't use loader to boot a kernel.
 874          */
 875         xsoftc.boot_delay = CAM_BOOT_DELAY;
 876 #endif
 877         /*
 878          * The xpt layer is, itself, the equivelent of a SIM.
 879          * Allow 16 ccbs in the ccb pool for it.  This should
 880          * give decent parallelism when we probe busses and
 881          * perform other XPT functions.
 882          */
 883         devq = cam_simq_alloc(16);
 884         xpt_sim = cam_sim_alloc(xptaction,
 885                                 xptpoll,
 886                                 "xpt",
 887                                 /*softc*/NULL,
 888                                 /*unit*/0,
 889                                 /*mtx*/&xsoftc.xpt_lock,
 890                                 /*max_dev_transactions*/0,
 891                                 /*max_tagged_dev_transactions*/0,
 892                                 devq);
 893         if (xpt_sim == NULL)
 894                 return (ENOMEM);
 895
 896         mtx_lock(&xsoftc.xpt_lock);
 897         if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
 898                 mtx_unlock(&xsoftc.xpt_lock);
 899                 printf("xpt_init: xpt_bus_register failed with status %#x,"
 900                        " failing attach\n", status);
 901                 return (EINVAL);
 902         }
 903
 904         /*
 905          * Looking at the XPT from the SIM layer, the XPT is
 906          * the equivelent of a peripheral driver.  Allocate
 907          * a peripheral driver entry for us.
 908          */
 909         if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
 910                                       CAM_TARGET_WILDCARD,
 911                                       CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
 912                 mtx_unlock(&xsoftc.xpt_lock);
 913                 printf("xpt_init: xpt_create_path failed with status %#x,"
 914                        " failing attach\n", status);
 915                 return (EINVAL);
 916         }
 917
 918         cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
 919                          path, NULL, 0, xpt_sim);
 920         xpt_free_path(path);
 921         mtx_unlock(&xsoftc.xpt_lock);
 922         if (cam_num_doneqs < 1)
 923                 cam_num_doneqs = 1 + mp_ncpus / 6;
 924         else if (cam_num_doneqs > MAXCPU)
 925                 cam_num_doneqs = MAXCPU;
 926         for (i = 0; i < cam_num_doneqs; i++) {
 927                 mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
 928                     MTX_DEF);
 929                 STAILQ_INIT(&cam_doneqs[i].cam_doneq);
 930                 error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
 931                     &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
 932                 if (error != 0) {
 933                         cam_num_doneqs = i;
 934                         break;
 935                 }
 936         }
 937         if (cam_num_doneqs < 1) {
 938                 printf("xpt_init: Cannot init completion queues "
 939                        "- failing attach\n");
 940                 return (ENOMEM);
 941         }
 942         /*
 943          * Register a callback for when interrupts are enabled.
 944          */
 945         xsoftc.xpt_config_hook =
 946             (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
 947                                               M_CAMXPT, M_NOWAIT | M_ZERO);
 948         if (xsoftc.xpt_config_hook == NULL) {
 949                 printf("xpt_init: Cannot malloc config hook "
 950                        "- failing attach\n");
 951                 return (ENOMEM);
 952         }
 953         xsoftc.xpt_config_hook->ich_func = xpt_config;
 954         if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
 955                 free (xsoftc.xpt_config_hook, M_CAMXPT);
 956                 printf("xpt_init: config_intrhook_establish failed "
 957                        "- failing attach\n");
 958         }
 959
 960         return (0);
 961 }
 962
 963 static cam_status
 964 xptregister(struct cam_periph *periph, void *arg)
 965 {
 966         struct cam_sim *xpt_sim;
 967
 968         if (periph == NULL) {
 969                 printf("xptregister: periph was NULL!!\n");
 970                 return(CAM_REQ_CMP_ERR);
 971         }
 972
 973         xpt_sim = (struct cam_sim *)arg;
 974         xpt_sim->softc = periph;
 975         xpt_periph = periph;
 976         periph->softc = NULL;
 977
 978         return(CAM_REQ_CMP);
 979 }
 980
 981 int32_t
 982 xpt_add_periph(struct cam_periph *periph)
 983 {
 984         struct cam_ed *device;
 985         int32_t  status;
 986
 987         TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
 988         device = periph->path->device;
 989         status = CAM_REQ_CMP;
 990         if (device != NULL) {
 991                 mtx_lock(&device->target->bus->eb_mtx);
 992                 device->generation++;
 993                 SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
 994                 mtx_unlock(&device->target->bus->eb_mtx);
 995         }
 996
 997         return (status);
 998 }
 999
1000 void
1001 xpt_remove_periph(struct cam_periph *periph)
1002 {
1003         struct cam_ed *device;
1004
1005         device = periph->path->device;
1006         if (device != NULL) {
1007                 mtx_lock(&device->target->bus->eb_mtx);
1008                 device->generation++;
1009                 SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1010                 mtx_unlock(&device->target->bus->eb_mtx);
1011         }
1012 }
1013
1014
1015 void
1016 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1017 {
1018         struct  cam_path *path = periph->path;
1019
1020         cam_periph_assert(periph, MA_OWNED);
1021         periph->flags |= CAM_PERIPH_ANNOUNCED;
1022
1023         printf("%s%d at %s%d bus %d scbus%d target %d lun %d\n",
1024                periph->periph_name, periph->unit_number,
1025                path->bus->sim->sim_name,
1026                path->bus->sim->unit_number,
1027                path->bus->sim->bus_id,
1028                path->bus->path_id,
1029                path->target->target_id,
1030                path->device->lun_id);
1031         printf("%s%d: ", periph->periph_name, periph->unit_number);
1032         if (path->device->protocol == PROTO_SCSI)
1033                 scsi_print_inquiry(&path->device->inq_data);
1034         else if (path->device->protocol == PROTO_ATA ||
1035             path->device->protocol == PROTO_SATAPM)
1036                 ata_print_ident(&path->device->ident_data);
1037         else if (path->device->protocol == PROTO_SEMB)
1038                 semb_print_ident(
1039                     (struct sep_identify_data *)&path->device->ident_data);
1040         else
1041                 printf("Unknown protocol device\n");
1042         if (path->device->serial_num_len > 0) {
1043                 /* Don't wrap the screen  - print only the first 60 chars */
1044                 printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1045                        periph->unit_number, path->device->serial_num);
1046         }
1047         /* Announce transport details. */
1048         (*(path->bus->xport->announce))(periph);
1049         /* Announce command queueing. */
1050         if (path->device->inq_flags & SID_CmdQue
1051          || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1052                 printf("%s%d: Command Queueing enabled\n",
1053                        periph->periph_name, periph->unit_number);
1054         }
1055         /* Announce caller's details if they've passed in. */
1056         if (announce_string != NULL)
1057                 printf("%s%d: %s\n", periph->periph_name,
1058                        periph->unit_number, announce_string);
1059 }
1060
1061 void
1062 xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1063 {
1064         if (quirks != 0) {
1065                 printf("%s%d: quirks=0x%b\n", periph->periph_name,
1066                     periph->unit_number, quirks, bit_string);
1067         }
1068 }
1069
1070 void
1071 xpt_denounce_periph(struct cam_periph *periph)
1072 {
1073         struct  cam_path *path = periph->path;
1074
1075         cam_periph_assert(periph, MA_OWNED);
1076         printf("%s%d at %s%d bus %d scbus%d target %d lun %d\n",
1077                periph->periph_name, periph->unit_number,
1078                path->bus->sim->sim_name,
1079                path->bus->sim->unit_number,
1080                path->bus->sim->bus_id,
1081                path->bus->path_id,
1082                path->target->target_id,
1083                path->device->lun_id);
1084         printf("%s%d: ", periph->periph_name, periph->unit_number);
1085         if (path->device->protocol == PROTO_SCSI)
1086                 scsi_print_inquiry_short(&path->device->inq_data);
1087         else if (path->device->protocol == PROTO_ATA ||
1088             path->device->protocol == PROTO_SATAPM)
1089                 ata_print_ident_short(&path->device->ident_data);
1090         else if (path->device->protocol == PROTO_SEMB)
1091                 semb_print_ident_short(
1092                     (struct sep_identify_data *)&path->device->ident_data);
1093         else
1094                 printf("Unknown protocol device");
1095         if (path->device->serial_num_len > 0)
1096                 printf(" s/n %.60s", path->device->serial_num);
1097         printf(" detached\n");
1098 }
1099
1100
1101 int
1102 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1103 {
1104         int ret = -1, l;
1105         struct ccb_dev_advinfo cdai;
1106         struct scsi_vpd_id_descriptor *idd;
1107
1108         xpt_path_assert(path, MA_OWNED);
1109
1110         memset(&cdai, 0, sizeof(cdai));
1111         xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1112         cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1113         cdai.bufsiz = len;
1114
1115         if (!strcmp(attr, "GEOM::ident"))
1116                 cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1117         else if (!strcmp(attr, "GEOM::physpath"))
1118                 cdai.buftype = CDAI_TYPE_PHYS_PATH;
1119         else if (strcmp(attr, "GEOM::lunid") == 0 ||
1120                  strcmp(attr, "GEOM::lunname") == 0) {
1121                 cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1122                 cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1123         } else
1124                 goto out;
1125
1126         cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1127         if (cdai.buf == NULL) {
1128                 ret = ENOMEM;
1129                 goto out;
1130         }
1131         xpt_action((union ccb *)&cdai); /* can only be synchronous */
1132         if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1133                 cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1134         if (cdai.provsiz == 0)
1135                 goto out;
1136         if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1137                 if (strcmp(attr, "GEOM::lunid") == 0) {
1138                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1139                             cdai.provsiz, scsi_devid_is_lun_naa);
1140                         if (idd == NULL)
1141                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1142                                     cdai.provsiz, scsi_devid_is_lun_eui64);
1143                 } else
1144                         idd = NULL;
1145                 if (idd == NULL)
1146                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1147                             cdai.provsiz, scsi_devid_is_lun_t10);
1148                 if (idd == NULL)
1149                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1150                             cdai.provsiz, scsi_devid_is_lun_name);
1151                 if (idd == NULL)
1152                         goto out;
1153                 ret = 0;
1154                 if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII ||
1155                     (idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1156                         l = strnlen(idd->identifier, idd->length);
1157                         if (l < len) {
1158                                 bcopy(idd->identifier, buf, l);
1159                                 buf[l] = 0;
1160                         } else
1161                                 ret = EFAULT;
1162                 } else {
1163                         if (idd->length * 2 < len) {
1164                                 for (l = 0; l < idd->length; l++)
1165                                         sprintf(buf + l * 2, "%02x",
1166                                             idd->identifier[l]);
1167                         } else
1168                                 ret = EFAULT;
1169                 }
1170         } else {
1171                 ret = 0;
1172                 if (strlcpy(buf, cdai.buf, len) >= len)
1173                         ret = EFAULT;
1174         }
1175
1176 out:
1177         if (cdai.buf != NULL)
1178                 free(cdai.buf, M_CAMXPT);
1179         return ret;
1180 }
1181
1182 static dev_match_ret
1183 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1184             struct cam_eb *bus)
1185 {
1186         dev_match_ret retval;
1187         int i;
1188
1189         retval = DM_RET_NONE;
1190
1191         /*
1192          * If we aren't given something to match against, that's an error.
1193          */
1194         if (bus == NULL)
1195                 return(DM_RET_ERROR);
1196
1197         /*
1198          * If there are no match entries, then this bus matches no
1199          * matter what.
1200          */
1201         if ((patterns == NULL) || (num_patterns == 0))
1202                 return(DM_RET_DESCEND | DM_RET_COPY);
1203
1204         for (i = 0; i < num_patterns; i++) {
1205                 struct bus_match_pattern *cur_pattern;
1206
1207                 /*
1208                  * If the pattern in question isn't for a bus node, we
1209                  * aren't interested.  However, we do indicate to the
1210                  * calling routine that we should continue descending the
1211                  * tree, since the user wants to match against lower-level
1212                  * EDT elements.
1213                  */
1214                 if (patterns[i].type != DEV_MATCH_BUS) {
1215                         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1216                                 retval |= DM_RET_DESCEND;
1217                         continue;
1218                 }
1219
1220                 cur_pattern = &patterns[i].pattern.bus_pattern;
1221
1222                 /*
1223                  * If they want to match any bus node, we give them any
1224                  * device node.
1225                  */
1226                 if (cur_pattern->flags == BUS_MATCH_ANY) {
1227                         /* set the copy flag */
1228                         retval |= DM_RET_COPY;
1229
1230                         /*
1231                          * If we've already decided on an action, go ahead
1232                          * and return.
1233                          */
1234                         if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1235                                 return(retval);
1236                 }
1237
1238                 /*
1239                  * Not sure why someone would do this...
1240                  */
1241                 if (cur_pattern->flags == BUS_MATCH_NONE)
1242                         continue;
1243
1244                 if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1245                  && (cur_pattern->path_id != bus->path_id))
1246                         continue;
1247
1248                 if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1249                  && (cur_pattern->bus_id != bus->sim->bus_id))
1250                         continue;
1251
1252                 if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1253                  && (cur_pattern->unit_number != bus->sim->unit_number))
1254                         continue;
1255
1256                 if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1257                  && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1258                              DEV_IDLEN) != 0))
1259                         continue;
1260
1261                 /*
1262                  * If we get to this point, the user definitely wants
1263                  * information on this bus.  So tell the caller to copy the
1264                  * data out.
1265                  */
1266                 retval |= DM_RET_COPY;
1267
1268                 /*
1269                  * If the return action has been set to descend, then we
1270                  * know that we've already seen a non-bus matching
1271                  * expression, therefore we need to further descend the tree.
1272                  * This won't change by continuing around the loop, so we
1273                  * go ahead and return.  If we haven't seen a non-bus
1274                  * matching expression, we keep going around the loop until
1275                  * we exhaust the matching expressions.  We'll set the stop
1276                  * flag once we fall out of the loop.
1277                  */
1278                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1279                         return(retval);
1280         }
1281
1282         /*
1283          * If the return action hasn't been set to descend yet, that means
1284          * we haven't seen anything other than bus matching patterns.  So
1285          * tell the caller to stop descending the tree -- the user doesn't
1286          * want to match against lower level tree elements.
1287          */
1288         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1289                 retval |= DM_RET_STOP;
1290
1291         return(retval);
1292 }
1293
1294 static dev_match_ret
1295 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1296                struct cam_ed *device)
1297 {
1298         dev_match_ret retval;
1299         int i;
1300
1301         retval = DM_RET_NONE;
1302
1303         /*
1304          * If we aren't given something to match against, that's an error.
1305          */
1306         if (device == NULL)
1307                 return(DM_RET_ERROR);
1308
1309         /*
1310          * If there are no match entries, then this device matches no
1311          * matter what.
1312          */
1313         if ((patterns == NULL) || (num_patterns == 0))
1314                 return(DM_RET_DESCEND | DM_RET_COPY);
1315
1316         for (i = 0; i < num_patterns; i++) {
1317                 struct device_match_pattern *cur_pattern;
1318                 struct scsi_vpd_device_id *device_id_page;
1319
1320                 /*
1321                  * If the pattern in question isn't for a device node, we
1322                  * aren't interested.
1323                  */
1324                 if (patterns[i].type != DEV_MATCH_DEVICE) {
1325                         if ((patterns[i].type == DEV_MATCH_PERIPH)
1326                          && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1327                                 retval |= DM_RET_DESCEND;
1328                         continue;
1329                 }
1330
1331                 cur_pattern = &patterns[i].pattern.device_pattern;
1332
1333                 /* Error out if mutually exclusive options are specified. */
1334                 if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1335                  == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1336                         return(DM_RET_ERROR);
1337
1338                 /*
1339                  * If they want to match any device node, we give them any
1340                  * device node.
1341                  */
1342                 if (cur_pattern->flags == DEV_MATCH_ANY)
1343                         goto copy_dev_node;
1344
1345                 /*
1346                  * Not sure why someone would do this...
1347                  */
1348                 if (cur_pattern->flags == DEV_MATCH_NONE)
1349                         continue;
1350
1351                 if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1352                  && (cur_pattern->path_id != device->target->bus->path_id))
1353                         continue;
1354
1355                 if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1356                  && (cur_pattern->target_id != device->target->target_id))
1357                         continue;
1358
1359                 if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1360                  && (cur_pattern->target_lun != device->lun_id))
1361                         continue;
1362
1363                 if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1364                  && (cam_quirkmatch((caddr_t)&device->inq_data,
1365                                     (caddr_t)&cur_pattern->data.inq_pat,
1366                                     1, sizeof(cur_pattern->data.inq_pat),
1367                                     scsi_static_inquiry_match) == NULL))
1368                         continue;
1369
1370                 device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1371                 if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1372                  && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1373                   || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1374                                       device->device_id_len
1375                                     - SVPD_DEVICE_ID_HDR_LEN,
1376                                       cur_pattern->data.devid_pat.id,
1377                                       cur_pattern->data.devid_pat.id_len) != 0))
1378                         continue;
1379
1380 copy_dev_node:
1381                 /*
1382                  * If we get to this point, the user definitely wants
1383                  * information on this device.  So tell the caller to copy
1384                  * the data out.
1385                  */
1386                 retval |= DM_RET_COPY;
1387
1388                 /*
1389                  * If the return action has been set to descend, then we
1390                  * know that we've already seen a peripheral matching
1391                  * expression, therefore we need to further descend the tree.
1392                  * This won't change by continuing around the loop, so we
1393                  * go ahead and return.  If we haven't seen a peripheral
1394                  * matching expression, we keep going around the loop until
1395                  * we exhaust the matching expressions.  We'll set the stop
1396                  * flag once we fall out of the loop.
1397                  */
1398                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1399                         return(retval);
1400         }
1401
1402         /*
1403          * If the return action hasn't been set to descend yet, that means
1404          * we haven't seen any peripheral matching patterns.  So tell the
1405          * caller to stop descending the tree -- the user doesn't want to
1406          * match against lower level tree elements.
1407          */
1408         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1409                 retval |= DM_RET_STOP;
1410
1411         return(retval);
1412 }
1413
1414 /*
1415  * Match a single peripheral against any number of match patterns.
1416  */
1417 static dev_match_ret
1418 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1419                struct cam_periph *periph)
1420 {
1421         dev_match_ret retval;
1422         int i;
1423
1424         /*
1425          * If we aren't given something to match against, that's an error.
1426          */
1427         if (periph == NULL)
1428                 return(DM_RET_ERROR);
1429
1430         /*
1431          * If there are no match entries, then this peripheral matches no
1432          * matter what.
1433          */
1434         if ((patterns == NULL) || (num_patterns == 0))
1435                 return(DM_RET_STOP | DM_RET_COPY);
1436
1437         /*
1438          * There aren't any nodes below a peripheral node, so there's no
1439          * reason to descend the tree any further.
1440          */
1441         retval = DM_RET_STOP;
1442
1443         for (i = 0; i < num_patterns; i++) {
1444                 struct periph_match_pattern *cur_pattern;
1445
1446                 /*
1447                  * If the pattern in question isn't for a peripheral, we
1448                  * aren't interested.
1449                  */
1450                 if (patterns[i].type != DEV_MATCH_PERIPH)
1451                         continue;
1452
1453                 cur_pattern = &patterns[i].pattern.periph_pattern;
1454
1455                 /*
1456                  * If they want to match on anything, then we will do so.
1457                  */
1458                 if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1459                         /* set the copy flag */
1460                         retval |= DM_RET_COPY;
1461
1462                         /*
1463                          * We've already set the return action to stop,
1464                          * since there are no nodes below peripherals in
1465                          * the tree.
1466                          */
1467                         return(retval);
1468                 }
1469
1470                 /*
1471                  * Not sure why someone would do this...
1472                  */
1473                 if (cur_pattern->flags == PERIPH_MATCH_NONE)
1474                         continue;
1475
1476                 if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1477                  && (cur_pattern->path_id != periph->path->bus->path_id))
1478                         continue;
1479
1480                 /*
1481                  * For the target and lun id's, we have to make sure the
1482                  * target and lun pointers aren't NULL.  The xpt peripheral
1483                  * has a wildcard target and device.
1484                  */
1485                 if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1486                  && ((periph->path->target == NULL)
1487                  ||(cur_pattern->target_id != periph->path->target->target_id)))
1488                         continue;
1489
1490                 if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1491                  && ((periph->path->device == NULL)
1492                  || (cur_pattern->target_lun != periph->path->device->lun_id)))
1493                         continue;
1494
1495                 if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1496                  && (cur_pattern->unit_number != periph->unit_number))
1497                         continue;
1498
1499                 if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1500                  && (strncmp(cur_pattern->periph_name, periph->periph_name,
1501                              DEV_IDLEN) != 0))
1502                         continue;
1503
1504                 /*
1505                  * If we get to this point, the user definitely wants
1506                  * information on this peripheral.  So tell the caller to
1507                  * copy the data out.
1508                  */
1509                 retval |= DM_RET_COPY;
1510
1511                 /*
1512                  * The return action has already been set to stop, since
1513                  * peripherals don't have any nodes below them in the EDT.
1514                  */
1515                 return(retval);
1516         }
1517
1518         /*
1519          * If we get to this point, the peripheral that was passed in
1520          * doesn't match any of the patterns.
1521          */
1522         return(retval);
1523 }
1524
1525 static int
1526 xptedtbusfunc(struct cam_eb *bus, void *arg)
1527 {
1528         struct ccb_dev_match *cdm;
1529         struct cam_et *target;
1530         dev_match_ret retval;
1531
1532         cdm = (struct ccb_dev_match *)arg;
1533
1534         /*
1535          * If our position is for something deeper in the tree, that means
1536          * that we've already seen this node.  So, we keep going down.
1537          */
1538         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1539          && (cdm->pos.cookie.bus == bus)
1540          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1541          && (cdm->pos.cookie.target != NULL))
1542                 retval = DM_RET_DESCEND;
1543         else
1544                 retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1545
1546         /*
1547          * If we got an error, bail out of the search.
1548          */
1549         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1550                 cdm->status = CAM_DEV_MATCH_ERROR;
1551                 return(0);
1552         }
1553
1554         /*
1555          * If the copy flag is set, copy this bus out.
1556          */
1557         if (retval & DM_RET_COPY) {
1558                 int spaceleft, j;
1559
1560                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1561                         sizeof(struct dev_match_result));
1562
1563                 /*
1564                  * If we don't have enough space to put in another
1565                  * match result, save our position and tell the
1566                  * user there are more devices to check.
1567                  */
1568                 if (spaceleft < sizeof(struct dev_match_result)) {
1569                         bzero(&cdm->pos, sizeof(cdm->pos));
1570                         cdm->pos.position_type =
1571                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1572
1573                         cdm->pos.cookie.bus = bus;
1574                         cdm->pos.generations[CAM_BUS_GENERATION]=
1575                                 xsoftc.bus_generation;
1576                         cdm->status = CAM_DEV_MATCH_MORE;
1577                         return(0);
1578                 }
1579                 j = cdm->num_matches;
1580                 cdm->num_matches++;
1581                 cdm->matches[j].type = DEV_MATCH_BUS;
1582                 cdm->matches[j].result.bus_result.path_id = bus->path_id;
1583                 cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1584                 cdm->matches[j].result.bus_result.unit_number =
1585                         bus->sim->unit_number;
1586                 strncpy(cdm->matches[j].result.bus_result.dev_name,
1587                         bus->sim->sim_name, DEV_IDLEN);
1588         }
1589
1590         /*
1591          * If the user is only interested in busses, there's no
1592          * reason to descend to the next level in the tree.
1593          */
1594         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1595                 return(1);
1596
1597         /*
1598          * If there is a target generation recorded, check it to
1599          * make sure the target list hasn't changed.
1600          */
1601         mtx_lock(&bus->eb_mtx);
1602         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1603          && (cdm->pos.cookie.bus == bus)
1604          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1605          && (cdm->pos.cookie.target != NULL)) {
1606                 if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1607                     bus->generation)) {
1608                         mtx_unlock(&bus->eb_mtx);
1609                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1610                         return (0);
1611                 }
1612                 target = (struct cam_et *)cdm->pos.cookie.target;
1613                 target->refcount++;
1614         } else
1615                 target = NULL;
1616         mtx_unlock(&bus->eb_mtx);
1617
1618         return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1619 }
1620
1621 static int
1622 xptedttargetfunc(struct cam_et *target, void *arg)
1623 {
1624         struct ccb_dev_match *cdm;
1625         struct cam_eb *bus;
1626         struct cam_ed *device;
1627
1628         cdm = (struct ccb_dev_match *)arg;
1629         bus = target->bus;
1630
1631         /*
1632          * If there is a device list generation recorded, check it to
1633          * make sure the device list hasn't changed.
1634          */
1635         mtx_lock(&bus->eb_mtx);
1636         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1637          && (cdm->pos.cookie.bus == bus)
1638          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1639          && (cdm->pos.cookie.target == target)
1640          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1641          && (cdm->pos.cookie.device != NULL)) {
1642                 if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1643                     target->generation) {
1644                         mtx_unlock(&bus->eb_mtx);
1645                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1646                         return(0);
1647                 }
1648                 device = (struct cam_ed *)cdm->pos.cookie.device;
1649                 device->refcount++;
1650         } else
1651                 device = NULL;
1652         mtx_unlock(&bus->eb_mtx);
1653
1654         return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1655 }
1656
1657 static int
1658 xptedtdevicefunc(struct cam_ed *device, void *arg)
1659 {
1660         struct cam_eb *bus;
1661         struct cam_periph *periph;
1662         struct ccb_dev_match *cdm;
1663         dev_match_ret retval;
1664
1665         cdm = (struct ccb_dev_match *)arg;
1666         bus = device->target->bus;
1667
1668         /*
1669          * If our position is for something deeper in the tree, that means
1670          * that we've already seen this node.  So, we keep going down.
1671          */
1672         if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1673          && (cdm->pos.cookie.device == device)
1674          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1675          && (cdm->pos.cookie.periph != NULL))
1676                 retval = DM_RET_DESCEND;
1677         else
1678                 retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1679                                         device);
1680
1681         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1682                 cdm->status = CAM_DEV_MATCH_ERROR;
1683                 return(0);
1684         }
1685
1686         /*
1687          * If the copy flag is set, copy this device out.
1688          */
1689         if (retval & DM_RET_COPY) {
1690                 int spaceleft, j;
1691
1692                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1693                         sizeof(struct dev_match_result));
1694
1695                 /*
1696                  * If we don't have enough space to put in another
1697                  * match result, save our position and tell the
1698                  * user there are more devices to check.
1699                  */
1700                 if (spaceleft < sizeof(struct dev_match_result)) {
1701                         bzero(&cdm->pos, sizeof(cdm->pos));
1702                         cdm->pos.position_type =
1703                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1704                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1705
1706                         cdm->pos.cookie.bus = device->target->bus;
1707                         cdm->pos.generations[CAM_BUS_GENERATION]=
1708                                 xsoftc.bus_generation;
1709                         cdm->pos.cookie.target = device->target;
1710                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1711                                 device->target->bus->generation;
1712                         cdm->pos.cookie.device = device;
1713                         cdm->pos.generations[CAM_DEV_GENERATION] =
1714                                 device->target->generation;
1715                         cdm->status = CAM_DEV_MATCH_MORE;
1716                         return(0);
1717                 }
1718                 j = cdm->num_matches;
1719                 cdm->num_matches++;
1720                 cdm->matches[j].type = DEV_MATCH_DEVICE;
1721                 cdm->matches[j].result.device_result.path_id =
1722                         device->target->bus->path_id;
1723                 cdm->matches[j].result.device_result.target_id =
1724                         device->target->target_id;
1725                 cdm->matches[j].result.device_result.target_lun =
1726                         device->lun_id;
1727                 cdm->matches[j].result.device_result.protocol =
1728                         device->protocol;
1729                 bcopy(&device->inq_data,
1730                       &cdm->matches[j].result.device_result.inq_data,
1731                       sizeof(struct scsi_inquiry_data));
1732                 bcopy(&device->ident_data,
1733                       &cdm->matches[j].result.device_result.ident_data,
1734                       sizeof(struct ata_params));
1735
1736                 /* Let the user know whether this device is unconfigured */
1737                 if (device->flags & CAM_DEV_UNCONFIGURED)
1738                         cdm->matches[j].result.device_result.flags =
1739                                 DEV_RESULT_UNCONFIGURED;
1740                 else
1741                         cdm->matches[j].result.device_result.flags =
1742                                 DEV_RESULT_NOFLAG;
1743         }
1744
1745         /*
1746          * If the user isn't interested in peripherals, don't descend
1747          * the tree any further.
1748          */
1749         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1750                 return(1);
1751
1752         /*
1753          * If there is a peripheral list generation recorded, make sure
1754          * it hasn't changed.
1755          */
1756         xpt_lock_buses();
1757         mtx_lock(&bus->eb_mtx);
1758         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1759          && (cdm->pos.cookie.bus == bus)
1760          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1761          && (cdm->pos.cookie.target == device->target)
1762          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1763          && (cdm->pos.cookie.device == device)
1764          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1765          && (cdm->pos.cookie.periph != NULL)) {
1766                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1767                     device->generation) {
1768                         mtx_unlock(&bus->eb_mtx);
1769                         xpt_unlock_buses();
1770                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1771                         return(0);
1772                 }
1773                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1774                 periph->refcount++;
1775         } else
1776                 periph = NULL;
1777         mtx_unlock(&bus->eb_mtx);
1778         xpt_unlock_buses();
1779
1780         return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1781 }
1782
1783 static int
1784 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1785 {
1786         struct ccb_dev_match *cdm;
1787         dev_match_ret retval;
1788
1789         cdm = (struct ccb_dev_match *)arg;
1790
1791         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1792
1793         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1794                 cdm->status = CAM_DEV_MATCH_ERROR;
1795                 return(0);
1796         }
1797
1798         /*
1799          * If the copy flag is set, copy this peripheral out.
1800          */
1801         if (retval & DM_RET_COPY) {
1802                 int spaceleft, j;
1803
1804                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1805                         sizeof(struct dev_match_result));
1806
1807                 /*
1808                  * If we don't have enough space to put in another
1809                  * match result, save our position and tell the
1810                  * user there are more devices to check.
1811                  */
1812                 if (spaceleft < sizeof(struct dev_match_result)) {
1813                         bzero(&cdm->pos, sizeof(cdm->pos));
1814                         cdm->pos.position_type =
1815                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1816                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1817                                 CAM_DEV_POS_PERIPH;
1818
1819                         cdm->pos.cookie.bus = periph->path->bus;
1820                         cdm->pos.generations[CAM_BUS_GENERATION]=
1821                                 xsoftc.bus_generation;
1822                         cdm->pos.cookie.target = periph->path->target;
1823                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1824                                 periph->path->bus->generation;
1825                         cdm->pos.cookie.device = periph->path->device;
1826                         cdm->pos.generations[CAM_DEV_GENERATION] =
1827                                 periph->path->target->generation;
1828                         cdm->pos.cookie.periph = periph;
1829                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
1830                                 periph->path->device->generation;
1831                         cdm->status = CAM_DEV_MATCH_MORE;
1832                         return(0);
1833                 }
1834
1835                 j = cdm->num_matches;
1836                 cdm->num_matches++;
1837                 cdm->matches[j].type = DEV_MATCH_PERIPH;
1838                 cdm->matches[j].result.periph_result.path_id =
1839                         periph->path->bus->path_id;
1840                 cdm->matches[j].result.periph_result.target_id =
1841                         periph->path->target->target_id;
1842                 cdm->matches[j].result.periph_result.target_lun =
1843                         periph->path->device->lun_id;
1844                 cdm->matches[j].result.periph_result.unit_number =
1845                         periph->unit_number;
1846                 strncpy(cdm->matches[j].result.periph_result.periph_name,
1847                         periph->periph_name, DEV_IDLEN);
1848         }
1849
1850         return(1);
1851 }
1852
1853 static int
1854 xptedtmatch(struct ccb_dev_match *cdm)
1855 {
1856         struct cam_eb *bus;
1857         int ret;
1858
1859         cdm->num_matches = 0;
1860
1861         /*
1862          * Check the bus list generation.  If it has changed, the user
1863          * needs to reset everything and start over.
1864          */
1865         xpt_lock_buses();
1866         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1867          && (cdm->pos.cookie.bus != NULL)) {
1868                 if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1869                     xsoftc.bus_generation) {
1870                         xpt_unlock_buses();
1871                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1872                         return(0);
1873                 }
1874                 bus = (struct cam_eb *)cdm->pos.cookie.bus;
1875                 bus->refcount++;
1876         } else
1877                 bus = NULL;
1878         xpt_unlock_buses();
1879
1880         ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1881
1882         /*
1883          * If we get back 0, that means that we had to stop before fully
1884          * traversing the EDT.  It also means that one of the subroutines
1885          * has set the status field to the proper value.  If we get back 1,
1886          * we've fully traversed the EDT and copied out any matching entries.
1887          */
1888         if (ret == 1)
1889                 cdm->status = CAM_DEV_MATCH_LAST;
1890
1891         return(ret);
1892 }
1893
1894 static int
1895 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1896 {
1897         struct cam_periph *periph;
1898         struct ccb_dev_match *cdm;
1899
1900         cdm = (struct ccb_dev_match *)arg;
1901
1902         xpt_lock_buses();
1903         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1904          && (cdm->pos.cookie.pdrv == pdrv)
1905          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1906          && (cdm->pos.cookie.periph != NULL)) {
1907                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1908                     (*pdrv)->generation) {
1909                         xpt_unlock_buses();
1910                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1911                         return(0);
1912                 }
1913                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1914                 periph->refcount++;
1915         } else
1916                 periph = NULL;
1917         xpt_unlock_buses();
1918
1919         return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1920 }
1921
1922 static int
1923 xptplistperiphfunc(struct cam_periph *periph, void *arg)
1924 {
1925         struct ccb_dev_match *cdm;
1926         dev_match_ret retval;
1927
1928         cdm = (struct ccb_dev_match *)arg;
1929
1930         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1931
1932         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1933                 cdm->status = CAM_DEV_MATCH_ERROR;
1934                 return(0);
1935         }
1936
1937         /*
1938          * If the copy flag is set, copy this peripheral out.
1939          */
1940         if (retval & DM_RET_COPY) {
1941                 int spaceleft, j;
1942
1943                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1944                         sizeof(struct dev_match_result));
1945
1946                 /*
1947                  * If we don't have enough space to put in another
1948                  * match result, save our position and tell the
1949                  * user there are more devices to check.
1950                  */
1951                 if (spaceleft < sizeof(struct dev_match_result)) {
1952                         struct periph_driver **pdrv;
1953
1954                         pdrv = NULL;
1955                         bzero(&cdm->pos, sizeof(cdm->pos));
1956                         cdm->pos.position_type =
1957                                 CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1958                                 CAM_DEV_POS_PERIPH;
1959
1960                         /*
1961                          * This may look a bit non-sensical, but it is
1962                          * actually quite logical.  There are very few
1963                          * peripheral drivers, and bloating every peripheral
1964                          * structure with a pointer back to its parent
1965                          * peripheral driver linker set entry would cost
1966                          * more in the long run than doing this quick lookup.
1967                          */
1968                         for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1969                                 if (strcmp((*pdrv)->driver_name,
1970                                     periph->periph_name) == 0)
1971                                         break;
1972                         }
1973
1974                         if (*pdrv == NULL) {
1975                                 cdm->status = CAM_DEV_MATCH_ERROR;
1976                                 return(0);
1977                         }
1978
1979                         cdm->pos.cookie.pdrv = pdrv;
1980                         /*
1981                          * The periph generation slot does double duty, as
1982                          * does the periph pointer slot.  They are used for
1983                          * both edt and pdrv lookups and positioning.
1984                          */
1985                         cdm->pos.cookie.periph = periph;
1986                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
1987                                 (*pdrv)->generation;
1988                         cdm->status = CAM_DEV_MATCH_MORE;
1989                         return(0);
1990                 }
1991
1992                 j = cdm->num_matches;
1993                 cdm->num_matches++;
1994                 cdm->matches[j].type = DEV_MATCH_PERIPH;
1995                 cdm->matches[j].result.periph_result.path_id =
1996                         periph->path->bus->path_id;
1997
1998                 /*
1999                  * The transport layer peripheral doesn't have a target or
2000                  * lun.
2001                  */
2002                 if (periph->path->target)
2003                         cdm->matches[j].result.periph_result.target_id =
2004                                 periph->path->target->target_id;
2005                 else
2006                         cdm->matches[j].result.periph_result.target_id = -1;
2007
2008                 if (periph->path->device)
2009                         cdm->matches[j].result.periph_result.target_lun =
2010                                 periph->path->device->lun_id;
2011                 else
2012                         cdm->matches[j].result.periph_result.target_lun = -1;
2013
2014                 cdm->matches[j].result.periph_result.unit_number =
2015                         periph->unit_number;
2016                 strncpy(cdm->matches[j].result.periph_result.periph_name,
2017                         periph->periph_name, DEV_IDLEN);
2018         }
2019
2020         return(1);
2021 }
2022
2023 static int
2024 xptperiphlistmatch(struct ccb_dev_match *cdm)
2025 {
2026         int ret;
2027
2028         cdm->num_matches = 0;
2029
2030         /*
2031          * At this point in the edt traversal function, we check the bus
2032          * list generation to make sure that no busses have been added or
2033          * removed since the user last sent a XPT_DEV_MATCH ccb through.
2034          * For the peripheral driver list traversal function, however, we
2035          * don't have to worry about new peripheral driver types coming or
2036          * going; they're in a linker set, and therefore can't change
2037          * without a recompile.
2038          */
2039
2040         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2041          && (cdm->pos.cookie.pdrv != NULL))
2042                 ret = xptpdrvtraverse(
2043                                 (struct periph_driver **)cdm->pos.cookie.pdrv,
2044                                 xptplistpdrvfunc, cdm);
2045         else
2046                 ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2047
2048         /*
2049          * If we get back 0, that means that we had to stop before fully
2050          * traversing the peripheral driver tree.  It also means that one of
2051          * the subroutines has set the status field to the proper value.  If
2052          * we get back 1, we've fully traversed the EDT and copied out any
2053          * matching entries.
2054          */
2055         if (ret == 1)
2056                 cdm->status = CAM_DEV_MATCH_LAST;
2057
2058         return(ret);
2059 }
2060
2061 static int
2062 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2063 {
2064         struct cam_eb *bus, *next_bus;
2065         int retval;
2066
2067         retval = 1;
2068         if (start_bus)
2069                 bus = start_bus;
2070         else {
2071                 xpt_lock_buses();
2072                 bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2073                 if (bus == NULL) {
2074                         xpt_unlock_buses();
2075                         return (retval);
2076                 }
2077                 bus->refcount++;
2078                 xpt_unlock_buses();
2079         }
2080         for (; bus != NULL; bus = next_bus) {
2081                 retval = tr_func(bus, arg);
2082                 if (retval == 0) {
2083                         xpt_release_bus(bus);
2084                         break;
2085                 }
2086                 xpt_lock_buses();
2087                 next_bus = TAILQ_NEXT(bus, links);
2088                 if (next_bus)
2089                         next_bus->refcount++;
2090                 xpt_unlock_buses();
2091                 xpt_release_bus(bus);
2092         }
2093         return(retval);
2094 }
2095
2096 static int
2097 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2098                   xpt_targetfunc_t *tr_func, void *arg)
2099 {
2100         struct cam_et *target, *next_target;
2101         int retval;
2102
2103         retval = 1;
2104         if (start_target)
2105                 target = start_target;
2106         else {
2107                 mtx_lock(&bus->eb_mtx);
2108                 target = TAILQ_FIRST(&bus->et_entries);
2109                 if (target == NULL) {
2110                         mtx_unlock(&bus->eb_mtx);
2111                         return (retval);
2112                 }
2113                 target->refcount++;
2114                 mtx_unlock(&bus->eb_mtx);
2115         }
2116         for (; target != NULL; target = next_target) {
2117                 retval = tr_func(target, arg);
2118                 if (retval == 0) {
2119                         xpt_release_target(target);
2120                         break;
2121                 }
2122                 mtx_lock(&bus->eb_mtx);
2123                 next_target = TAILQ_NEXT(target, links);
2124                 if (next_target)
2125                         next_target->refcount++;
2126                 mtx_unlock(&bus->eb_mtx);
2127                 xpt_release_target(target);
2128         }
2129         return(retval);
2130 }
2131
2132 static int
2133 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2134                   xpt_devicefunc_t *tr_func, void *arg)
2135 {
2136         struct cam_eb *bus;
2137         struct cam_ed *device, *next_device;
2138         int retval;
2139
2140         retval = 1;
2141         bus = target->bus;
2142         if (start_device)
2143                 device = start_device;
2144         else {
2145                 mtx_lock(&bus->eb_mtx);
2146                 device = TAILQ_FIRST(&target->ed_entries);
2147                 if (device == NULL) {
2148                         mtx_unlock(&bus->eb_mtx);
2149                         return (retval);
2150                 }
2151                 device->refcount++;
2152                 mtx_unlock(&bus->eb_mtx);
2153         }
2154         for (; device != NULL; device = next_device) {
2155                 mtx_lock(&device->device_mtx);
2156                 retval = tr_func(device, arg);
2157                 mtx_unlock(&device->device_mtx);
2158                 if (retval == 0) {
2159                         xpt_release_device(device);
2160                         break;
2161                 }
2162                 mtx_lock(&bus->eb_mtx);
2163                 next_device = TAILQ_NEXT(device, links);
2164                 if (next_device)
2165                         next_device->refcount++;
2166                 mtx_unlock(&bus->eb_mtx);
2167                 xpt_release_device(device);
2168         }
2169         return(retval);
2170 }
2171
2172 static int
2173 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2174                   xpt_periphfunc_t *tr_func, void *arg)
2175 {
2176         struct cam_eb *bus;
2177         struct cam_periph *periph, *next_periph;
2178         int retval;
2179
2180         retval = 1;
2181
2182         bus = device->target->bus;
2183         if (start_periph)
2184                 periph = start_periph;
2185         else {
2186                 xpt_lock_buses();
2187                 mtx_lock(&bus->eb_mtx);
2188                 periph = SLIST_FIRST(&device->periphs);
2189                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2190                         periph = SLIST_NEXT(periph, periph_links);
2191                 if (periph == NULL) {
2192                         mtx_unlock(&bus->eb_mtx);
2193                         xpt_unlock_buses();
2194                         return (retval);
2195                 }
2196                 periph->refcount++;
2197                 mtx_unlock(&bus->eb_mtx);
2198                 xpt_unlock_buses();
2199         }
2200         for (; periph != NULL; periph = next_periph) {
2201                 retval = tr_func(periph, arg);
2202                 if (retval == 0) {
2203                         cam_periph_release(periph);
2204                         break;
2205                 }
2206                 xpt_lock_buses();
2207                 mtx_lock(&bus->eb_mtx);
2208                 next_periph = SLIST_NEXT(periph, periph_links);
2209                 while (next_periph != NULL &&
2210                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2211                         next_periph = SLIST_NEXT(periph, periph_links);
2212                 if (next_periph)
2213                         next_periph->refcount++;
2214                 mtx_unlock(&bus->eb_mtx);
2215                 xpt_unlock_buses();
2216                 cam_periph_release_locked(periph);
2217         }
2218         return(retval);
2219 }
2220
2221 static int
2222 xptpdrvtraverse(struct periph_driver **start_pdrv,
2223                 xpt_pdrvfunc_t *tr_func, void *arg)
2224 {
2225         struct periph_driver **pdrv;
2226         int retval;
2227
2228         retval = 1;
2229
2230         /*
2231          * We don't traverse the peripheral driver list like we do the
2232          * other lists, because it is a linker set, and therefore cannot be
2233          * changed during runtime.  If the peripheral driver list is ever
2234          * re-done to be something other than a linker set (i.e. it can
2235          * change while the system is running), the list traversal should
2236          * be modified to work like the other traversal functions.
2237          */
2238         for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2239              *pdrv != NULL; pdrv++) {
2240                 retval = tr_func(pdrv, arg);
2241
2242                 if (retval == 0)
2243                         return(retval);
2244         }
2245
2246         return(retval);
2247 }
2248
2249 static int
2250 xptpdperiphtraverse(struct periph_driver **pdrv,
2251                     struct cam_periph *start_periph,
2252                     xpt_periphfunc_t *tr_func, void *arg)
2253 {
2254         struct cam_periph *periph, *next_periph;
2255         int retval;
2256
2257         retval = 1;
2258
2259         if (start_periph)
2260                 periph = start_periph;
2261         else {
2262                 xpt_lock_buses();
2263                 periph = TAILQ_FIRST(&(*pdrv)->units);
2264                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2265                         periph = TAILQ_NEXT(periph, unit_links);
2266                 if (periph == NULL) {
2267                         xpt_unlock_buses();
2268                         return (retval);
2269                 }
2270                 periph->refcount++;
2271                 xpt_unlock_buses();
2272         }
2273         for (; periph != NULL; periph = next_periph) {
2274                 cam_periph_lock(periph);
2275                 retval = tr_func(periph, arg);
2276                 cam_periph_unlock(periph);
2277                 if (retval == 0) {
2278                         cam_periph_release(periph);
2279                         break;
2280                 }
2281                 xpt_lock_buses();
2282                 next_periph = TAILQ_NEXT(periph, unit_links);
2283                 while (next_periph != NULL &&
2284                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2285                         next_periph = TAILQ_NEXT(periph, unit_links);
2286                 if (next_periph)
2287                         next_periph->refcount++;
2288                 xpt_unlock_buses();
2289                 cam_periph_release(periph);
2290         }
2291         return(retval);
2292 }
2293
2294 static int
2295 xptdefbusfunc(struct cam_eb *bus, void *arg)
2296 {
2297         struct xpt_traverse_config *tr_config;
2298
2299         tr_config = (struct xpt_traverse_config *)arg;
2300
2301         if (tr_config->depth == XPT_DEPTH_BUS) {
2302                 xpt_busfunc_t *tr_func;
2303
2304                 tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2305
2306                 return(tr_func(bus, tr_config->tr_arg));
2307         } else
2308                 return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2309 }
2310
2311 static int
2312 xptdeftargetfunc(struct cam_et *target, void *arg)
2313 {
2314         struct xpt_traverse_config *tr_config;
2315
2316         tr_config = (struct xpt_traverse_config *)arg;
2317
2318         if (tr_config->depth == XPT_DEPTH_TARGET) {
2319                 xpt_targetfunc_t *tr_func;
2320
2321                 tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2322
2323                 return(tr_func(target, tr_config->tr_arg));
2324         } else
2325                 return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2326 }
2327
2328 static int
2329 xptdefdevicefunc(struct cam_ed *device, void *arg)
2330 {
2331         struct xpt_traverse_config *tr_config;
2332
2333         tr_config = (struct xpt_traverse_config *)arg;
2334
2335         if (tr_config->depth == XPT_DEPTH_DEVICE) {
2336                 xpt_devicefunc_t *tr_func;
2337
2338                 tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2339
2340                 return(tr_func(device, tr_config->tr_arg));
2341         } else
2342                 return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2343 }
2344
2345 static int
2346 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2347 {
2348         struct xpt_traverse_config *tr_config;
2349         xpt_periphfunc_t *tr_func;
2350
2351         tr_config = (struct xpt_traverse_config *)arg;
2352
2353         tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2354
2355         /*
2356          * Unlike the other default functions, we don't check for depth
2357          * here.  The peripheral driver level is the last level in the EDT,
2358          * so if we're here, we should execute the function in question.
2359          */
2360         return(tr_func(periph, tr_config->tr_arg));
2361 }
2362
2363 /*
2364  * Execute the given function for every bus in the EDT.
2365  */
2366 static int
2367 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2368 {
2369         struct xpt_traverse_config tr_config;
2370
2371         tr_config.depth = XPT_DEPTH_BUS;
2372         tr_config.tr_func = tr_func;
2373         tr_config.tr_arg = arg;
2374
2375         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2376 }
2377
2378 /*
2379  * Execute the given function for every device in the EDT.
2380  */
2381 static int
2382 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2383 {
2384         struct xpt_traverse_config tr_config;
2385
2386         tr_config.depth = XPT_DEPTH_DEVICE;
2387         tr_config.tr_func = tr_func;
2388         tr_config.tr_arg = arg;
2389
2390         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2391 }
2392
2393 static int
2394 xptsetasyncfunc(struct cam_ed *device, void *arg)
2395 {
2396         struct cam_path path;
2397         struct ccb_getdev cgd;
2398         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2399
2400         /*
2401          * Don't report unconfigured devices (Wildcard devs,
2402          * devices only for target mode, device instances
2403          * that have been invalidated but are waiting for
2404          * their last reference count to be released).
2405          */
2406         if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2407                 return (1);
2408
2409         xpt_compile_path(&path,
2410                          NULL,
2411                          device->target->bus->path_id,
2412                          device->target->target_id,
2413                          device->lun_id);
2414         xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2415         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2416         xpt_action((union ccb *)&cgd);
2417         csa->callback(csa->callback_arg,
2418                             AC_FOUND_DEVICE,
2419                             &path, &cgd);
2420         xpt_release_path(&path);
2421
2422         return(1);
2423 }
2424
2425 static int
2426 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2427 {
2428         struct cam_path path;
2429         struct ccb_pathinq cpi;
2430         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2431
2432         xpt_compile_path(&path, /*periph*/NULL,
2433                          bus->path_id,
2434                          CAM_TARGET_WILDCARD,
2435                          CAM_LUN_WILDCARD);
2436         xpt_path_lock(&path);
2437         xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2438         cpi.ccb_h.func_code = XPT_PATH_INQ;
2439         xpt_action((union ccb *)&cpi);
2440         csa->callback(csa->callback_arg,
2441                             AC_PATH_REGISTERED,
2442                             &path, &cpi);
2443         xpt_path_unlock(&path);
2444         xpt_release_path(&path);
2445
2446         return(1);
2447 }
2448
2449 void
2450 xpt_action(union ccb *start_ccb)
2451 {
2452
2453         CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2454
2455         start_ccb->ccb_h.status = CAM_REQ_INPROG;
2456         (*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2457 }
2458
2459 void
2460 xpt_action_default(union ccb *start_ccb)
2461 {
2462         struct cam_path *path;
2463         struct cam_sim *sim;
2464         int lock;
2465
2466         path = start_ccb->ccb_h.path;
2467         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_action_default\n"));
2468
2469         switch (start_ccb->ccb_h.func_code) {
2470         case XPT_SCSI_IO:
2471         {
2472                 struct cam_ed *device;
2473
2474                 /*
2475                  * For the sake of compatibility with SCSI-1
2476                  * devices that may not understand the identify
2477                  * message, we include lun information in the
2478                  * second byte of all commands.  SCSI-1 specifies
2479                  * that luns are a 3 bit value and reserves only 3
2480                  * bits for lun information in the CDB.  Later
2481                  * revisions of the SCSI spec allow for more than 8
2482                  * luns, but have deprecated lun information in the
2483                  * CDB.  So, if the lun won't fit, we must omit.
2484                  *
2485                  * Also be aware that during initial probing for devices,
2486                  * the inquiry information is unknown but initialized to 0.
2487                  * This means that this code will be exercised while probing
2488                  * devices with an ANSI revision greater than 2.
2489                  */
2490                 device = path->device;
2491                 if (device->protocol_version <= SCSI_REV_2
2492                  && start_ccb->ccb_h.target_lun < 8
2493                  && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2494
2495                         start_ccb->csio.cdb_io.cdb_bytes[1] |=
2496                             start_ccb->ccb_h.target_lun << 5;
2497                 }
2498                 start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2499         }
2500         /* FALLTHROUGH */
2501         case XPT_TARGET_IO:
2502         case XPT_CONT_TARGET_IO:
2503                 start_ccb->csio.sense_resid = 0;
2504                 start_ccb->csio.resid = 0;
2505                 /* FALLTHROUGH */
2506         case XPT_ATA_IO:
2507                 if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2508                         start_ccb->ataio.resid = 0;
2509                 /* FALLTHROUGH */
2510         case XPT_RESET_DEV:
2511         case XPT_ENG_EXEC:
2512         case XPT_SMP_IO:
2513         {
2514                 struct cam_devq *devq;
2515
2516                 devq = path->bus->sim->devq;
2517                 mtx_lock(&devq->send_mtx);
2518                 cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2519                 if (xpt_schedule_devq(devq, path->device) != 0)
2520                         xpt_run_devq(devq);
2521                 mtx_unlock(&devq->send_mtx);
2522                 break;
2523         }
2524         case XPT_CALC_GEOMETRY:
2525                 /* Filter out garbage */
2526                 if (start_ccb->ccg.block_size == 0
2527                  || start_ccb->ccg.volume_size == 0) {
2528                         start_ccb->ccg.cylinders = 0;
2529                         start_ccb->ccg.heads = 0;
2530                         start_ccb->ccg.secs_per_track = 0;
2531                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2532                         break;
2533                 }
2534 #if defined(PC98) || defined(__sparc64__)
2535                 /*
2536                  * In a PC-98 system, geometry translation depens on
2537                  * the "real" device geometry obtained from mode page 4.
2538                  * SCSI geometry translation is performed in the
2539                  * initialization routine of the SCSI BIOS and the result
2540                  * stored in host memory.  If the translation is available
2541                  * in host memory, use it.  If not, rely on the default
2542                  * translation the device driver performs.
2543                  * For sparc64, we may need adjust the geometry of large
2544                  * disks in order to fit the limitations of the 16-bit
2545                  * fields of the VTOC8 disk label.
2546                  */
2547                 if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2548                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2549                         break;
2550                 }
2551 #endif
2552                 goto call_sim;
2553         case XPT_ABORT:
2554         {
2555                 union ccb* abort_ccb;
2556
2557                 abort_ccb = start_ccb->cab.abort_ccb;
2558                 if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2559
2560                         if (abort_ccb->ccb_h.pinfo.index >= 0) {
2561                                 struct cam_ccbq *ccbq;
2562                                 struct cam_ed *device;
2563
2564                                 device = abort_ccb->ccb_h.path->device;
2565                                 ccbq = &device->ccbq;
2566                                 cam_ccbq_remove_ccb(ccbq, abort_ccb);
2567                                 abort_ccb->ccb_h.status =
2568                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2569                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2570                                 xpt_done(abort_ccb);
2571                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2572                                 break;
2573                         }
2574                         if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2575                          && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2576                                 /*
2577                                  * We've caught this ccb en route to
2578                                  * the SIM.  Flag it for abort and the
2579                                  * SIM will do so just before starting
2580                                  * real work on the CCB.
2581                                  */
2582                                 abort_ccb->ccb_h.status =
2583                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2584                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2585                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2586                                 break;
2587                         }
2588                 }
2589                 if (XPT_FC_IS_QUEUED(abort_ccb)
2590                  && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2591                         /*
2592                          * It's already completed but waiting
2593                          * for our SWI to get to it.
2594                          */
2595                         start_ccb->ccb_h.status = CAM_UA_ABORT;
2596                         break;
2597                 }
2598                 /*
2599                  * If we weren't able to take care of the abort request
2600                  * in the XPT, pass the request down to the SIM for processing.
2601                  */
2602         }
2603         /* FALLTHROUGH */
2604         case XPT_ACCEPT_TARGET_IO:
2605         case XPT_EN_LUN:
2606         case XPT_IMMED_NOTIFY:
2607         case XPT_NOTIFY_ACK:
2608         case XPT_RESET_BUS:
2609         case XPT_IMMEDIATE_NOTIFY:
2610         case XPT_NOTIFY_ACKNOWLEDGE:
2611         case XPT_GET_SIM_KNOB:
2612         case XPT_SET_SIM_KNOB:
2613         case XPT_GET_TRAN_SETTINGS:
2614         case XPT_SET_TRAN_SETTINGS:
2615         case XPT_PATH_INQ:
2616 call_sim:
2617                 sim = path->bus->sim;
2618                 lock = (mtx_owned(sim->mtx) == 0);
2619                 if (lock)
2620                         CAM_SIM_LOCK(sim);
2621                 (*(sim->sim_action))(sim, start_ccb);
2622                 if (lock)
2623                         CAM_SIM_UNLOCK(sim);
2624                 break;
2625         case XPT_PATH_STATS:
2626                 start_ccb->cpis.last_reset = path->bus->last_reset;
2627                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2628                 break;
2629         case XPT_GDEV_TYPE:
2630         {
2631                 struct cam_ed *dev;
2632
2633                 dev = path->device;
2634                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2635                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2636                 } else {
2637                         struct ccb_getdev *cgd;
2638
2639                         cgd = &start_ccb->cgd;
2640                         cgd->protocol = dev->protocol;
2641                         cgd->inq_data = dev->inq_data;
2642                         cgd->ident_data = dev->ident_data;
2643                         cgd->inq_flags = dev->inq_flags;
2644                         cgd->ccb_h.status = CAM_REQ_CMP;
2645                         cgd->serial_num_len = dev->serial_num_len;
2646                         if ((dev->serial_num_len > 0)
2647                          && (dev->serial_num != NULL))
2648                                 bcopy(dev->serial_num, cgd->serial_num,
2649                                       dev->serial_num_len);
2650                 }
2651                 break;
2652         }
2653         case XPT_GDEV_STATS:
2654         {
2655                 struct cam_ed *dev;
2656
2657                 dev = path->device;
2658                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2659                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2660                 } else {
2661                         struct ccb_getdevstats *cgds;
2662                         struct cam_eb *bus;
2663                         struct cam_et *tar;
2664
2665                         cgds = &start_ccb->cgds;
2666                         bus = path->bus;
2667                         tar = path->target;
2668                         cgds->dev_openings = dev->ccbq.dev_openings;
2669                         cgds->dev_active = dev->ccbq.dev_active;
2670                         cgds->devq_openings = dev->ccbq.devq_openings;
2671                         cgds->devq_queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2672                         cgds->held = dev->ccbq.held;
2673                         cgds->last_reset = tar->last_reset;
2674                         cgds->maxtags = dev->maxtags;
2675                         cgds->mintags = dev->mintags;
2676                         if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2677                                 cgds->last_reset = bus->last_reset;
2678                         cgds->ccb_h.status = CAM_REQ_CMP;
2679                 }
2680                 break;
2681         }
2682         case XPT_GDEVLIST:
2683         {
2684                 struct cam_periph       *nperiph;
2685                 struct periph_list      *periph_head;
2686                 struct ccb_getdevlist   *cgdl;
2687                 u_int                   i;
2688                 struct cam_ed           *device;
2689                 int                     found;
2690
2691
2692                 found = 0;
2693
2694                 /*
2695                  * Don't want anyone mucking with our data.
2696                  */
2697                 device = path->device;
2698                 periph_head = &device->periphs;
2699                 cgdl = &start_ccb->cgdl;
2700
2701                 /*
2702                  * Check and see if the list has changed since the user
2703                  * last requested a list member.  If so, tell them that the
2704                  * list has changed, and therefore they need to start over
2705                  * from the beginning.
2706                  */
2707                 if ((cgdl->index != 0) &&
2708                     (cgdl->generation != device->generation)) {
2709                         cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2710                         break;
2711                 }
2712
2713                 /*
2714                  * Traverse the list of peripherals and attempt to find
2715                  * the requested peripheral.
2716                  */
2717                 for (nperiph = SLIST_FIRST(periph_head), i = 0;
2718                      (nperiph != NULL) && (i <= cgdl->index);
2719                      nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2720                         if (i == cgdl->index) {
2721                                 strncpy(cgdl->periph_name,
2722                                         nperiph->periph_name,
2723                                         DEV_IDLEN);
2724                                 cgdl->unit_number = nperiph->unit_number;
2725                                 found = 1;
2726                         }
2727                 }
2728                 if (found == 0) {
2729                         cgdl->status = CAM_GDEVLIST_ERROR;
2730                         break;
2731                 }
2732
2733                 if (nperiph == NULL)
2734                         cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2735                 else
2736                         cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2737
2738                 cgdl->index++;
2739                 cgdl->generation = device->generation;
2740
2741                 cgdl->ccb_h.status = CAM_REQ_CMP;
2742                 break;
2743         }
2744         case XPT_DEV_MATCH:
2745         {
2746                 dev_pos_type position_type;
2747                 struct ccb_dev_match *cdm;
2748
2749                 cdm = &start_ccb->cdm;
2750
2751                 /*
2752                  * There are two ways of getting at information in the EDT.
2753                  * The first way is via the primary EDT tree.  It starts
2754                  * with a list of busses, then a list of targets on a bus,
2755                  * then devices/luns on a target, and then peripherals on a
2756                  * device/lun.  The "other" way is by the peripheral driver
2757                  * lists.  The peripheral driver lists are organized by
2758                  * peripheral driver.  (obviously)  So it makes sense to
2759                  * use the peripheral driver list if the user is looking
2760                  * for something like "da1", or all "da" devices.  If the
2761                  * user is looking for something on a particular bus/target
2762                  * or lun, it's generally better to go through the EDT tree.
2763                  */
2764
2765                 if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2766                         position_type = cdm->pos.position_type;
2767                 else {
2768                         u_int i;
2769
2770                         position_type = CAM_DEV_POS_NONE;
2771
2772                         for (i = 0; i < cdm->num_patterns; i++) {
2773                                 if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2774                                  ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2775                                         position_type = CAM_DEV_POS_EDT;
2776                                         break;
2777                                 }
2778                         }
2779
2780                         if (cdm->num_patterns == 0)
2781                                 position_type = CAM_DEV_POS_EDT;
2782                         else if (position_type == CAM_DEV_POS_NONE)
2783                                 position_type = CAM_DEV_POS_PDRV;
2784                 }
2785
2786                 switch(position_type & CAM_DEV_POS_TYPEMASK) {
2787                 case CAM_DEV_POS_EDT:
2788                         xptedtmatch(cdm);
2789                         break;
2790                 case CAM_DEV_POS_PDRV:
2791                         xptperiphlistmatch(cdm);
2792                         break;
2793                 default:
2794                         cdm->status = CAM_DEV_MATCH_ERROR;
2795                         break;
2796                 }
2797
2798                 if (cdm->status == CAM_DEV_MATCH_ERROR)
2799                         start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2800                 else
2801                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2802
2803                 break;
2804         }
2805         case XPT_SASYNC_CB:
2806         {
2807                 struct ccb_setasync *csa;
2808                 struct async_node *cur_entry;
2809                 struct async_list *async_head;
2810                 u_int32_t added;
2811
2812                 csa = &start_ccb->csa;
2813                 added = csa->event_enable;
2814                 async_head = &path->device->asyncs;
2815
2816                 /*
2817                  * If there is already an entry for us, simply
2818                  * update it.
2819                  */
2820                 cur_entry = SLIST_FIRST(async_head);
2821                 while (cur_entry != NULL) {
2822                         if ((cur_entry->callback_arg == csa->callback_arg)
2823                          && (cur_entry->callback == csa->callback))
2824                                 break;
2825                         cur_entry = SLIST_NEXT(cur_entry, links);
2826                 }
2827
2828                 if (cur_entry != NULL) {
2829                         /*
2830                          * If the request has no flags set,
2831                          * remove the entry.
2832                          */
2833                         added &= ~cur_entry->event_enable;
2834                         if (csa->event_enable == 0) {
2835                                 SLIST_REMOVE(async_head, cur_entry,
2836                                              async_node, links);
2837                                 xpt_release_device(path->device);
2838                                 free(cur_entry, M_CAMXPT);
2839                         } else {
2840                                 cur_entry->event_enable = csa->event_enable;
2841                         }
2842                         csa->event_enable = added;
2843                 } else {
2844                         cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2845                                            M_NOWAIT);
2846                         if (cur_entry == NULL) {
2847                                 csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2848                                 break;
2849                         }
2850                         cur_entry->event_enable = csa->event_enable;
2851                         cur_entry->event_lock =
2852                             mtx_owned(path->bus->sim->mtx) ? 1 : 0;
2853                         cur_entry->callback_arg = csa->callback_arg;
2854                         cur_entry->callback = csa->callback;
2855                         SLIST_INSERT_HEAD(async_head, cur_entry, links);
2856                         xpt_acquire_device(path->device);
2857                 }
2858                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2859                 break;
2860         }
2861         case XPT_REL_SIMQ:
2862         {
2863                 struct ccb_relsim *crs;
2864                 struct cam_ed *dev;
2865
2866                 crs = &start_ccb->crs;
2867                 dev = path->device;
2868                 if (dev == NULL) {
2869
2870                         crs->ccb_h.status = CAM_DEV_NOT_THERE;
2871                         break;
2872                 }
2873
2874                 if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2875
2876                         /* Don't ever go below one opening */
2877                         if (crs->openings > 0) {
2878                                 xpt_dev_ccbq_resize(path, crs->openings);
2879                                 if (bootverbose) {
2880                                         xpt_print(path,
2881                                             "number of openings is now %d\n",
2882                                             crs->openings);
2883                                 }
2884                         }
2885                 }
2886
2887                 mtx_lock(&dev->sim->devq->send_mtx);
2888                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2889
2890                         if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2891
2892                                 /*
2893                                  * Just extend the old timeout and decrement
2894                                  * the freeze count so that a single timeout
2895                                  * is sufficient for releasing the queue.
2896                                  */
2897                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2898                                 callout_stop(&dev->callout);
2899                         } else {
2900
2901                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2902                         }
2903
2904                         callout_reset(&dev->callout,
2905                             (crs->release_timeout * hz) / 1000,
2906                             xpt_release_devq_timeout, dev);
2907
2908                         dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2909
2910                 }
2911
2912                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2913
2914                         if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2915                                 /*
2916                                  * Decrement the freeze count so that a single
2917                                  * completion is still sufficient to unfreeze
2918                                  * the queue.
2919                                  */
2920                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2921                         } else {
2922
2923                                 dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2924                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2925                         }
2926                 }
2927
2928                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2929
2930                         if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2931                          || (dev->ccbq.dev_active == 0)) {
2932
2933                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2934                         } else {
2935
2936                                 dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2937                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2938                         }
2939                 }
2940                 mtx_unlock(&dev->sim->devq->send_mtx);
2941
2942                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2943                         xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2944                 start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2945                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2946                 break;
2947         }
2948         case XPT_DEBUG: {
2949                 struct cam_path *oldpath;
2950
2951                 /* Check that all request bits are supported. */
2952                 if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
2953                         start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2954                         break;
2955                 }
2956
2957                 cam_dflags = CAM_DEBUG_NONE;
2958                 if (cam_dpath != NULL) {
2959                         oldpath = cam_dpath;
2960                         cam_dpath = NULL;
2961                         xpt_free_path(oldpath);
2962                 }
2963                 if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
2964                         if (xpt_create_path(&cam_dpath, NULL,
2965                                             start_ccb->ccb_h.path_id,
2966                                             start_ccb->ccb_h.target_id,
2967                                             start_ccb->ccb_h.target_lun) !=
2968                                             CAM_REQ_CMP) {
2969                                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2970                         } else {
2971                                 cam_dflags = start_ccb->cdbg.flags;
2972                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2973                                 xpt_print(cam_dpath, "debugging flags now %x\n",
2974                                     cam_dflags);
2975                         }
2976                 } else
2977                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2978                 break;
2979         }
2980         case XPT_NOOP:
2981                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
2982                         xpt_freeze_devq(path, 1);
2983                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2984                 break;
2985         default:
2986         case XPT_SDEV_TYPE:
2987         case XPT_TERM_IO:
2988         case XPT_ENG_INQ:
2989                 /* XXX Implement */
2990                 printf("%s: CCB type %#x not supported\n", __func__,
2991                        start_ccb->ccb_h.func_code);
2992                 start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
2993                 if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
2994                         xpt_done(start_ccb);
2995                 }
2996                 break;
2997         }
2998 }
2999
3000 void
3001 xpt_polled_action(union ccb *start_ccb)
3002 {
3003         u_int32_t timeout;
3004         struct    cam_sim *sim;
3005         struct    cam_devq *devq;
3006         struct    cam_ed *dev;
3007
3008         timeout = start_ccb->ccb_h.timeout * 10;
3009         sim = start_ccb->ccb_h.path->bus->sim;
3010         devq = sim->devq;
3011         dev = start_ccb->ccb_h.path->device;
3012
3013         mtx_unlock(&dev->device_mtx);
3014
3015         /*
3016          * Steal an opening so that no other queued requests
3017          * can get it before us while we simulate interrupts.
3018          */
3019         mtx_lock(&devq->send_mtx);
3020         dev->ccbq.devq_openings--;
3021         dev->ccbq.dev_openings--;
3022         while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3023             (--timeout > 0)) {
3024                 mtx_unlock(&devq->send_mtx);
3025                 DELAY(100);
3026                 CAM_SIM_LOCK(sim);
3027                 (*(sim->sim_poll))(sim);
3028                 CAM_SIM_UNLOCK(sim);
3029                 camisr_runqueue();
3030                 mtx_lock(&devq->send_mtx);
3031         }
3032         dev->ccbq.devq_openings++;
3033         dev->ccbq.dev_openings++;
3034         mtx_unlock(&devq->send_mtx);
3035
3036         if (timeout != 0) {
3037                 xpt_action(start_ccb);
3038                 while(--timeout > 0) {
3039                         CAM_SIM_LOCK(sim);
3040                         (*(sim->sim_poll))(sim);
3041                         CAM_SIM_UNLOCK(sim);
3042                         camisr_runqueue();
3043                         if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3044                             != CAM_REQ_INPROG)
3045                                 break;
3046                         DELAY(100);
3047                 }
3048                 if (timeout == 0) {
3049                         /*
3050                          * XXX Is it worth adding a sim_timeout entry
3051                          * point so we can attempt recovery?  If
3052                          * this is only used for dumps, I don't think
3053                          * it is.
3054                          */
3055                         start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3056                 }
3057         } else {
3058                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3059         }
3060
3061         mtx_lock(&dev->device_mtx);
3062 }
3063
3064 /*
3065  * Schedule a peripheral driver to receive a ccb when it's
3066  * target device has space for more transactions.
3067  */
3068 void
3069 xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3070 {
3071
3072         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3073         cam_periph_assert(periph, MA_OWNED);
3074         if (new_priority < periph->scheduled_priority) {
3075                 periph->scheduled_priority = new_priority;
3076                 xpt_run_allocq(periph, 0);
3077         }
3078 }
3079
3080
3081 /*
3082  * Schedule a device to run on a given queue.
3083  * If the device was inserted as a new entry on the queue,
3084  * return 1 meaning the device queue should be run. If we
3085  * were already queued, implying someone else has already
3086  * started the queue, return 0 so the caller doesn't attempt
3087  * to run the queue.
3088  */
3089 static int
3090 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3091                  u_int32_t new_priority)
3092 {
3093         int retval;
3094         u_int32_t old_priority;
3095
3096         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3097
3098         old_priority = pinfo->priority;
3099
3100         /*
3101          * Are we already queued?
3102          */
3103         if (pinfo->index != CAM_UNQUEUED_INDEX) {
3104                 /* Simply reorder based on new priority */
3105                 if (new_priority < old_priority) {
3106                         camq_change_priority(queue, pinfo->index,
3107                                              new_priority);
3108                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3109                                         ("changed priority to %d\n",
3110                                          new_priority));
3111                         retval = 1;
3112                 } else
3113                         retval = 0;
3114         } else {
3115                 /* New entry on the queue */
3116                 if (new_priority < old_priority)
3117                         pinfo->priority = new_priority;
3118
3119                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3120                                 ("Inserting onto queue\n"));
3121                 pinfo->generation = ++queue->generation;
3122                 camq_insert(queue, pinfo);
3123                 retval = 1;
3124         }
3125         return (retval);
3126 }
3127
3128 static void
3129 xpt_run_allocq_task(void *context, int pending)
3130 {
3131         struct cam_periph *periph = context;
3132
3133         cam_periph_lock(periph);
3134         periph->flags &= ~CAM_PERIPH_RUN_TASK;
3135         xpt_run_allocq(periph, 1);
3136         cam_periph_unlock(periph);
3137         cam_periph_release(periph);
3138 }
3139
3140 static void
3141 xpt_run_allocq(struct cam_periph *periph, int sleep)
3142 {
3143         struct cam_ed   *device;
3144         union ccb       *ccb;
3145         uint32_t         prio;
3146
3147         cam_periph_assert(periph, MA_OWNED);
3148         if (periph->periph_allocating)
3149                 return;
3150         periph->periph_allocating = 1;
3151         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3152         device = periph->path->device;
3153         ccb = NULL;
3154 restart:
3155         while ((prio = min(periph->scheduled_priority,
3156             periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3157             (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3158              device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3159
3160                 if (ccb == NULL &&
3161                     (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3162                         if (sleep) {
3163                                 ccb = xpt_get_ccb(periph);
3164                                 goto restart;
3165                         }
3166                         if (periph->flags & CAM_PERIPH_RUN_TASK)
3167                                 break;
3168                         xpt_lock_buses();
3169                         periph->refcount++;     /* Unconditionally acquire */
3170                         xpt_unlock_buses();
3171                         periph->flags |= CAM_PERIPH_RUN_TASK;
3172                         taskqueue_enqueue(xsoftc.xpt_taskq,
3173                             &periph->periph_run_task);
3174                         break;
3175                 }
3176                 xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3177                 if (prio == periph->immediate_priority) {
3178                         periph->immediate_priority = CAM_PRIORITY_NONE;
3179                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3180                                         ("waking cam_periph_getccb()\n"));
3181                         SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3182                                           periph_links.sle);
3183                         wakeup(&periph->ccb_list);
3184                 } else {
3185                         periph->scheduled_priority = CAM_PRIORITY_NONE;
3186                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3187                                         ("calling periph_start()\n"));
3188                         periph->periph_start(periph, ccb);
3189                 }
3190                 ccb = NULL;
3191         }
3192         if (ccb != NULL)
3193                 xpt_release_ccb(ccb);
3194         periph->periph_allocating = 0;
3195 }
3196
3197 static void
3198 xpt_run_devq(struct cam_devq *devq)
3199 {
3200         char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3201         int lock;
3202
3203         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3204
3205         devq->send_queue.qfrozen_cnt++;
3206         while ((devq->send_queue.entries > 0)
3207             && (devq->send_openings > 0)
3208             && (devq->send_queue.qfrozen_cnt <= 1)) {
3209                 struct  cam_ed *device;
3210                 union ccb *work_ccb;
3211                 struct  cam_sim *sim;
3212
3213                 device = (struct cam_ed *)camq_remove(&devq->send_queue,
3214                                                            CAMQ_HEAD);
3215                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3216                                 ("running device %p\n", device));
3217
3218                 work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3219                 if (work_ccb == NULL) {
3220                         printf("device on run queue with no ccbs???\n");
3221                         continue;
3222                 }
3223
3224                 if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3225
3226                         mtx_lock(&xsoftc.xpt_lock);
3227                         if (xsoftc.num_highpower <= 0) {
3228                                 /*
3229                                  * We got a high power command, but we
3230                                  * don't have any available slots.  Freeze
3231                                  * the device queue until we have a slot
3232                                  * available.
3233                                  */
3234                                 xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3235                                 STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3236                                                    highpowerq_entry);
3237
3238                                 mtx_unlock(&xsoftc.xpt_lock);
3239                                 continue;
3240                         } else {
3241                                 /*
3242                                  * Consume a high power slot while
3243                                  * this ccb runs.
3244                                  */
3245                                 xsoftc.num_highpower--;
3246                         }
3247                         mtx_unlock(&xsoftc.xpt_lock);
3248                 }
3249                 cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3250                 cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3251                 devq->send_openings--;
3252                 devq->send_active++;
3253                 xpt_schedule_devq(devq, device);
3254                 mtx_unlock(&devq->send_mtx);
3255
3256                 if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3257                         /*
3258                          * The client wants to freeze the queue
3259                          * after this CCB is sent.
3260                          */
3261                         xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3262                 }
3263
3264                 /* In Target mode, the peripheral driver knows best... */
3265                 if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3266                         if ((device->inq_flags & SID_CmdQue) != 0
3267                          && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3268                                 work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3269                         else
3270                                 /*
3271                                  * Clear this in case of a retried CCB that
3272                                  * failed due to a rejected tag.
3273                                  */
3274                                 work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3275                 }
3276
3277                 switch (work_ccb->ccb_h.func_code) {
3278                 case XPT_SCSI_IO:
3279                         CAM_DEBUG(work_ccb->ccb_h.path,
3280                             CAM_DEBUG_CDB,("%s. CDB: %s\n",
3281                              scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3282                                           &device->inq_data),
3283                              scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3284                                              cdb_str, sizeof(cdb_str))));
3285                         break;
3286                 case XPT_ATA_IO:
3287                         CAM_DEBUG(work_ccb->ccb_h.path,
3288                             CAM_DEBUG_CDB,("%s. ACB: %s\n",
3289                              ata_op_string(&work_ccb->ataio.cmd),
3290                              ata_cmd_string(&work_ccb->ataio.cmd,
3291                                             cdb_str, sizeof(cdb_str))));
3292                         break;
3293                 default:
3294                         break;
3295                 }
3296
3297                 /*
3298                  * Device queues can be shared among multiple SIM instances
3299                  * that reside on different busses.  Use the SIM from the
3300                  * queued device, rather than the one from the calling bus.
3301                  */
3302                 sim = device->sim;
3303                 lock = (mtx_owned(sim->mtx) == 0);
3304                 if (lock)
3305                         CAM_SIM_LOCK(sim);
3306                 (*(sim->sim_action))(sim, work_ccb);
3307                 if (lock)
3308                         CAM_SIM_UNLOCK(sim);
3309                 mtx_lock(&devq->send_mtx);
3310         }
3311         devq->send_queue.qfrozen_cnt--;
3312 }
3313
3314 /*
3315  * This function merges stuff from the slave ccb into the master ccb, while
3316  * keeping important fields in the master ccb constant.
3317  */
3318 void
3319 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3320 {
3321
3322         /*
3323          * Pull fields that are valid for peripheral drivers to set
3324          * into the master CCB along with the CCB "payload".
3325          */
3326         master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3327         master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3328         master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3329         master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3330         bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3331               sizeof(union ccb) - sizeof(struct ccb_hdr));
3332 }
3333
3334 void
3335 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3336 {
3337
3338         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3339         ccb_h->pinfo.priority = priority;
3340         ccb_h->path = path;
3341         ccb_h->path_id = path->bus->path_id;
3342         if (path->target)
3343                 ccb_h->target_id = path->target->target_id;
3344         else
3345                 ccb_h->target_id = CAM_TARGET_WILDCARD;
3346         if (path->device) {
3347                 ccb_h->target_lun = path->device->lun_id;
3348                 ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3349         } else {
3350                 ccb_h->target_lun = CAM_TARGET_WILDCARD;
3351         }
3352         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3353         ccb_h->flags = 0;
3354         ccb_h->xflags = 0;
3355 }
3356
3357 /* Path manipulation functions */
3358 cam_status
3359 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3360                 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3361 {
3362         struct     cam_path *path;
3363         cam_status status;
3364
3365         path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3366
3367         if (path == NULL) {
3368                 status = CAM_RESRC_UNAVAIL;
3369                 return(status);
3370         }
3371         status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3372         if (status != CAM_REQ_CMP) {
3373                 free(path, M_CAMPATH);
3374                 path = NULL;
3375         }
3376         *new_path_ptr = path;
3377         return (status);
3378 }
3379
3380 cam_status
3381 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3382                          struct cam_periph *periph, path_id_t path_id,
3383                          target_id_t target_id, lun_id_t lun_id)
3384 {
3385
3386         return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3387             lun_id));
3388 }
3389
3390 cam_status
3391 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3392                  path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3393 {
3394         struct       cam_eb *bus;
3395         struct       cam_et *target;
3396         struct       cam_ed *device;
3397         cam_status   status;
3398
3399         status = CAM_REQ_CMP;   /* Completed without error */
3400         target = NULL;          /* Wildcarded */
3401         device = NULL;          /* Wildcarded */
3402
3403         /*
3404          * We will potentially modify the EDT, so block interrupts
3405          * that may attempt to create cam paths.
3406          */
3407         bus = xpt_find_bus(path_id);
3408         if (bus == NULL) {
3409                 status = CAM_PATH_INVALID;
3410         } else {
3411                 xpt_lock_buses();
3412                 mtx_lock(&bus->eb_mtx);
3413                 target = xpt_find_target(bus, target_id);
3414                 if (target == NULL) {
3415                         /* Create one */
3416                         struct cam_et *new_target;
3417
3418                         new_target = xpt_alloc_target(bus, target_id);
3419                         if (new_target == NULL) {
3420                                 status = CAM_RESRC_UNAVAIL;
3421                         } else {
3422                                 target = new_target;
3423                         }
3424                 }
3425                 xpt_unlock_buses();
3426                 if (target != NULL) {
3427                         device = xpt_find_device(target, lun_id);
3428                         if (device == NULL) {
3429                                 /* Create one */
3430                                 struct cam_ed *new_device;
3431
3432                                 new_device =
3433                                     (*(bus->xport->alloc_device))(bus,
3434                                                                       target,
3435                                                                       lun_id);
3436                                 if (new_device == NULL) {
3437                                         status = CAM_RESRC_UNAVAIL;
3438                                 } else {
3439                                         device = new_device;
3440                                 }
3441                         }
3442                 }
3443                 mtx_unlock(&bus->eb_mtx);
3444         }
3445
3446         /*
3447          * Only touch the user's data if we are successful.
3448          */
3449         if (status == CAM_REQ_CMP) {
3450                 new_path->periph = perph;
3451                 new_path->bus = bus;
3452                 new_path->target = target;
3453                 new_path->device = device;
3454                 CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3455         } else {
3456                 if (device != NULL)
3457                         xpt_release_device(device);
3458                 if (target != NULL)
3459                         xpt_release_target(target);
3460                 if (bus != NULL)
3461                         xpt_release_bus(bus);
3462         }
3463         return (status);
3464 }
3465
3466 cam_status
3467 xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3468 {
3469         struct     cam_path *new_path;
3470
3471         new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3472         if (new_path == NULL)
3473                 return(CAM_RESRC_UNAVAIL);
3474         xpt_copy_path(new_path, path);
3475         *new_path_ptr = new_path;
3476         return (CAM_REQ_CMP);
3477 }
3478
3479 void
3480 xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3481 {
3482
3483         *new_path = *path;
3484         if (path->bus != NULL)
3485                 xpt_acquire_bus(path->bus);
3486         if (path->target != NULL)
3487                 xpt_acquire_target(path->target);
3488         if (path->device != NULL)
3489                 xpt_acquire_device(path->device);
3490 }
3491
3492 void
3493 xpt_release_path(struct cam_path *path)
3494 {
3495         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3496         if (path->device != NULL) {
3497                 xpt_release_device(path->device);
3498                 path->device = NULL;
3499         }
3500         if (path->target != NULL) {
3501                 xpt_release_target(path->target);
3502                 path->target = NULL;
3503         }
3504         if (path->bus != NULL) {
3505                 xpt_release_bus(path->bus);
3506                 path->bus = NULL;
3507         }
3508 }
3509
3510 void
3511 xpt_free_path(struct cam_path *path)
3512 {
3513
3514         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3515         xpt_release_path(path);
3516         free(path, M_CAMPATH);
3517 }
3518
3519 void
3520 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3521     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3522 {
3523
3524         xpt_lock_buses();
3525         if (bus_ref) {
3526                 if (path->bus)
3527                         *bus_ref = path->bus->refcount;
3528                 else
3529                         *bus_ref = 0;
3530         }
3531         if (periph_ref) {
3532                 if (path->periph)
3533                         *periph_ref = path->periph->refcount;
3534                 else
3535                         *periph_ref = 0;
3536         }
3537         xpt_unlock_buses();
3538         if (target_ref) {
3539                 if (path->target)
3540                         *target_ref = path->target->refcount;
3541                 else
3542                         *target_ref = 0;
3543         }
3544         if (device_ref) {
3545                 if (path->device)
3546                         *device_ref = path->device->refcount;
3547                 else
3548                         *device_ref = 0;
3549         }
3550 }
3551
3552 /*
3553  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3554  * in path1, 2 for match with wildcards in path2.
3555  */
3556 int
3557 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3558 {
3559         int retval = 0;
3560
3561         if (path1->bus != path2->bus) {
3562                 if (path1->bus->path_id == CAM_BUS_WILDCARD)
3563                         retval = 1;
3564                 else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3565                         retval = 2;
3566                 else
3567                         return (-1);
3568         }
3569         if (path1->target != path2->target) {
3570                 if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3571                         if (retval == 0)
3572                                 retval = 1;
3573                 } else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3574                         retval = 2;
3575                 else
3576                         return (-1);
3577         }
3578         if (path1->device != path2->device) {
3579                 if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3580                         if (retval == 0)
3581                                 retval = 1;
3582                 } else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3583                         retval = 2;
3584                 else
3585                         return (-1);
3586         }
3587         return (retval);
3588 }
3589
3590 int
3591 xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3592 {
3593         int retval = 0;
3594
3595         if (path->bus != dev->target->bus) {
3596                 if (path->bus->path_id == CAM_BUS_WILDCARD)
3597                         retval = 1;
3598                 else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3599                         retval = 2;
3600                 else
3601                         return (-1);
3602         }
3603         if (path->target != dev->target) {
3604                 if (path->target->target_id == CAM_TARGET_WILDCARD) {
3605                         if (retval == 0)
3606                                 retval = 1;
3607                 } else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3608                         retval = 2;
3609                 else
3610                         return (-1);
3611         }
3612         if (path->device != dev) {
3613                 if (path->device->lun_id == CAM_LUN_WILDCARD) {
3614                         if (retval == 0)
3615                                 retval = 1;
3616                 } else if (dev->lun_id == CAM_LUN_WILDCARD)
3617                         retval = 2;
3618                 else
3619                         return (-1);
3620         }
3621         return (retval);
3622 }
3623
3624 void
3625 xpt_print_path(struct cam_path *path)
3626 {
3627
3628         if (path == NULL)
3629                 printf("(nopath): ");
3630         else {
3631                 if (path->periph != NULL)
3632                         printf("(%s%d:", path->periph->periph_name,
3633                                path->periph->unit_number);
3634                 else
3635                         printf("(noperiph:");
3636
3637                 if (path->bus != NULL)
3638                         printf("%s%d:%d:", path->bus->sim->sim_name,
3639                                path->bus->sim->unit_number,
3640                                path->bus->sim->bus_id);
3641                 else
3642                         printf("nobus:");
3643
3644                 if (path->target != NULL)
3645                         printf("%d:", path->target->target_id);
3646                 else
3647                         printf("X:");
3648
3649                 if (path->device != NULL)
3650                         printf("%d): ", path->device->lun_id);
3651                 else
3652                         printf("X): ");
3653         }
3654 }
3655
3656 void
3657 xpt_print_device(struct cam_ed *device)
3658 {
3659
3660         if (device == NULL)
3661                 printf("(nopath): ");
3662         else {
3663                 printf("(noperiph:%s%d:%d:%d:%d): ", device->sim->sim_name,
3664                        device->sim->unit_number,
3665                        device->sim->bus_id,
3666                        device->target->target_id,
3667                        device->lun_id);
3668         }
3669 }
3670
3671 void
3672 xpt_print(struct cam_path *path, const char *fmt, ...)
3673 {
3674         va_list ap;
3675         xpt_print_path(path);
3676         va_start(ap, fmt);
3677         vprintf(fmt, ap);
3678         va_end(ap);
3679 }
3680
3681 int
3682 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3683 {
3684         struct sbuf sb;
3685
3686         sbuf_new(&sb, str, str_len, 0);
3687
3688         if (path == NULL)
3689                 sbuf_printf(&sb, "(nopath): ");
3690         else {
3691                 if (path->periph != NULL)
3692                         sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3693                                     path->periph->unit_number);
3694                 else
3695                         sbuf_printf(&sb, "(noperiph:");
3696
3697                 if (path->bus != NULL)
3698                         sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3699                                     path->bus->sim->unit_number,
3700                                     path->bus->sim->bus_id);
3701                 else
3702                         sbuf_printf(&sb, "nobus:");
3703
3704                 if (path->target != NULL)
3705                         sbuf_printf(&sb, "%d:", path->target->target_id);
3706                 else
3707                         sbuf_printf(&sb, "X:");
3708
3709                 if (path->device != NULL)
3710                         sbuf_printf(&sb, "%d): ", path->device->lun_id);
3711                 else
3712                         sbuf_printf(&sb, "X): ");
3713         }
3714         sbuf_finish(&sb);
3715
3716         return(sbuf_len(&sb));
3717 }
3718
3719 path_id_t
3720 xpt_path_path_id(struct cam_path *path)
3721 {
3722         return(path->bus->path_id);
3723 }
3724
3725 target_id_t
3726 xpt_path_target_id(struct cam_path *path)
3727 {
3728         if (path->target != NULL)
3729                 return (path->target->target_id);
3730         else
3731                 return (CAM_TARGET_WILDCARD);
3732 }
3733
3734 lun_id_t
3735 xpt_path_lun_id(struct cam_path *path)
3736 {
3737         if (path->device != NULL)
3738                 return (path->device->lun_id);
3739         else
3740                 return (CAM_LUN_WILDCARD);
3741 }
3742
3743 struct cam_sim *
3744 xpt_path_sim(struct cam_path *path)
3745 {
3746
3747         return (path->bus->sim);
3748 }
3749
3750 struct cam_periph*
3751 xpt_path_periph(struct cam_path *path)
3752 {
3753
3754         return (path->periph);
3755 }
3756
3757 int
3758 xpt_path_legacy_ata_id(struct cam_path *path)
3759 {
3760         struct cam_eb *bus;
3761         int bus_id;
3762
3763         if ((strcmp(path->bus->sim->sim_name, "ata") != 0) &&
3764             strcmp(path->bus->sim->sim_name, "ahcich") != 0 &&
3765             strcmp(path->bus->sim->sim_name, "mvsch") != 0 &&
3766             strcmp(path->bus->sim->sim_name, "siisch") != 0)
3767                 return (-1);
3768
3769         if (strcmp(path->bus->sim->sim_name, "ata") == 0 &&
3770             path->bus->sim->unit_number < 2) {
3771                 bus_id = path->bus->sim->unit_number;
3772         } else {
3773                 bus_id = 2;
3774                 xpt_lock_buses();
3775                 TAILQ_FOREACH(bus, &xsoftc.xpt_busses, links) {
3776                         if (bus == path->bus)
3777                                 break;
3778                         if ((strcmp(bus->sim->sim_name, "ata") == 0 &&
3779                              bus->sim->unit_number >= 2) ||
3780                             strcmp(bus->sim->sim_name, "ahcich") == 0 ||
3781                             strcmp(bus->sim->sim_name, "mvsch") == 0 ||
3782                             strcmp(bus->sim->sim_name, "siisch") == 0)
3783                                 bus_id++;
3784                 }
3785                 xpt_unlock_buses();
3786         }
3787         if (path->target != NULL) {
3788                 if (path->target->target_id < 2)
3789                         return (bus_id * 2 + path->target->target_id);
3790                 else
3791                         return (-1);
3792         } else
3793                 return (bus_id * 2);
3794 }
3795
3796 /*
3797  * Release a CAM control block for the caller.  Remit the cost of the structure
3798  * to the device referenced by the path.  If the this device had no 'credits'
3799  * and peripheral drivers have registered async callbacks for this notification
3800  * call them now.
3801  */
3802 void
3803 xpt_release_ccb(union ccb *free_ccb)
3804 {
3805         struct   cam_ed *device;
3806         struct   cam_periph *periph;
3807
3808         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3809         xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3810         device = free_ccb->ccb_h.path->device;
3811         periph = free_ccb->ccb_h.path->periph;
3812
3813         xpt_free_ccb(free_ccb);
3814         periph->periph_allocated--;
3815         cam_ccbq_release_opening(&device->ccbq);
3816         xpt_run_allocq(periph, 0);
3817 }
3818
3819 /* Functions accessed by SIM drivers */
3820
3821 static struct xpt_xport xport_default = {
3822         .alloc_device = xpt_alloc_device_default,
3823         .action = xpt_action_default,
3824         .async = xpt_dev_async_default,
3825 };
3826
3827 /*
3828  * A sim structure, listing the SIM entry points and instance
3829  * identification info is passed to xpt_bus_register to hook the SIM
3830  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3831  * for this new bus and places it in the array of busses and assigns
3832  * it a path_id.  The path_id may be influenced by "hard wiring"
3833  * information specified by the user.  Once interrupt services are
3834  * available, the bus will be probed.
3835  */
3836 int32_t
3837 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3838 {
3839         struct cam_eb *new_bus;
3840         struct cam_eb *old_bus;
3841         struct ccb_pathinq cpi;
3842         struct cam_path *path;
3843         cam_status status;
3844
3845         mtx_assert(sim->mtx, MA_OWNED);
3846
3847         sim->bus_id = bus;
3848         new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3849                                           M_CAMXPT, M_NOWAIT|M_ZERO);
3850         if (new_bus == NULL) {
3851                 /* Couldn't satisfy request */
3852                 return (CAM_RESRC_UNAVAIL);
3853         }
3854
3855         mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3856         TAILQ_INIT(&new_bus->et_entries);
3857         cam_sim_hold(sim);
3858         new_bus->sim = sim;
3859         timevalclear(&new_bus->last_reset);
3860         new_bus->flags = 0;
3861         new_bus->refcount = 1;  /* Held until a bus_deregister event */
3862         new_bus->generation = 0;
3863
3864         xpt_lock_buses();
3865         sim->path_id = new_bus->path_id =
3866             xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3867         old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3868         while (old_bus != NULL
3869             && old_bus->path_id < new_bus->path_id)
3870                 old_bus = TAILQ_NEXT(old_bus, links);
3871         if (old_bus != NULL)
3872                 TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3873         else
3874                 TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3875         xsoftc.bus_generation++;
3876         xpt_unlock_buses();
3877
3878         /*
3879          * Set a default transport so that a PATH_INQ can be issued to
3880          * the SIM.  This will then allow for probing and attaching of
3881          * a more appropriate transport.
3882          */
3883         new_bus->xport = &xport_default;
3884
3885         status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3886                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3887         if (status != CAM_REQ_CMP) {
3888                 xpt_release_bus(new_bus);
3889                 free(path, M_CAMXPT);
3890                 return (CAM_RESRC_UNAVAIL);
3891         }
3892
3893         xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3894         cpi.ccb_h.func_code = XPT_PATH_INQ;
3895         xpt_action((union ccb *)&cpi);
3896
3897         if (cpi.ccb_h.status == CAM_REQ_CMP) {
3898                 switch (cpi.transport) {
3899                 case XPORT_SPI:
3900                 case XPORT_SAS:
3901                 case XPORT_FC:
3902                 case XPORT_USB:
3903                 case XPORT_ISCSI:
3904                 case XPORT_SRP:
3905                 case XPORT_PPB:
3906                         new_bus->xport = scsi_get_xport();
3907                         break;
3908                 case XPORT_ATA:
3909                 case XPORT_SATA:
3910                         new_bus->xport = ata_get_xport();
3911                         break;
3912                 default:
3913                         new_bus->xport = &xport_default;
3914                         break;
3915                 }
3916         }
3917
3918         /* Notify interested parties */
3919         if (sim->path_id != CAM_XPT_PATH_ID) {
3920
3921                 xpt_async(AC_PATH_REGISTERED, path, &cpi);
3922                 if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3923                         union   ccb *scan_ccb;
3924
3925                         /* Initiate bus rescan. */
3926                         scan_ccb = xpt_alloc_ccb_nowait();
3927                         if (scan_ccb != NULL) {
3928                                 scan_ccb->ccb_h.path = path;
3929                                 scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3930                                 scan_ccb->crcn.flags = 0;
3931                                 xpt_rescan(scan_ccb);
3932                         } else
3933                                 xpt_print(path,
3934                                           "Can't allocate CCB to scan bus\n");
3935                 } else
3936                         xpt_free_path(path);
3937         } else
3938                 xpt_free_path(path);
3939         return (CAM_SUCCESS);
3940 }
3941
3942 int32_t
3943 xpt_bus_deregister(path_id_t pathid)
3944 {
3945         struct cam_path bus_path;
3946         cam_status status;
3947
3948         status = xpt_compile_path(&bus_path, NULL, pathid,
3949                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3950         if (status != CAM_REQ_CMP)
3951                 return (status);
3952
3953         xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3954         xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3955
3956         /* Release the reference count held while registered. */
3957         xpt_release_bus(bus_path.bus);
3958         xpt_release_path(&bus_path);
3959
3960         return (CAM_REQ_CMP);
3961 }
3962
3963 static path_id_t
3964 xptnextfreepathid(void)
3965 {
3966         struct cam_eb *bus;
3967         path_id_t pathid;
3968         const char *strval;
3969
3970         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
3971         pathid = 0;
3972         bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3973 retry:
3974         /* Find an unoccupied pathid */
3975         while (bus != NULL && bus->path_id <= pathid) {
3976                 if (bus->path_id == pathid)
3977                         pathid++;
3978                 bus = TAILQ_NEXT(bus, links);
3979         }
3980
3981         /*
3982          * Ensure that this pathid is not reserved for
3983          * a bus that may be registered in the future.
3984          */
3985         if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
3986                 ++pathid;
3987                 /* Start the search over */
3988                 goto retry;
3989         }
3990         return (pathid);
3991 }
3992
3993 static path_id_t
3994 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
3995 {
3996         path_id_t pathid;
3997         int i, dunit, val;
3998         char buf[32];
3999         const char *dname;
4000
4001         pathid = CAM_XPT_PATH_ID;
4002         snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4003         if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4004                 return (pathid);
4005         i = 0;
4006         while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4007                 if (strcmp(dname, "scbus")) {
4008                         /* Avoid a bit of foot shooting. */
4009                         continue;
4010                 }
4011                 if (dunit < 0)          /* unwired?! */
4012                         continue;
4013                 if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4014                         if (sim_bus == val) {
4015                                 pathid = dunit;
4016                                 break;
4017                         }
4018                 } else if (sim_bus == 0) {
4019                         /* Unspecified matches bus 0 */
4020                         pathid = dunit;
4021                         break;
4022                 } else {
4023                         printf("Ambiguous scbus configuration for %s%d "
4024                                "bus %d, cannot wire down.  The kernel "
4025                                "config entry for scbus%d should "
4026                                "specify a controller bus.\n"
4027                                "Scbus will be assigned dynamically.\n",
4028                                sim_name, sim_unit, sim_bus, dunit);
4029                         break;
4030                 }
4031         }
4032
4033         if (pathid == CAM_XPT_PATH_ID)
4034                 pathid = xptnextfreepathid();
4035         return (pathid);
4036 }
4037
4038 static const char *
4039 xpt_async_string(u_int32_t async_code)
4040 {
4041
4042         switch (async_code) {
4043         case AC_BUS_RESET: return ("AC_BUS_RESET");
4044         case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4045         case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4046         case AC_SENT_BDR: return ("AC_SENT_BDR");
4047         case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4048         case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4049         case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4050         case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4051         case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4052         case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4053         case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4054         case AC_CONTRACT: return ("AC_CONTRACT");
4055         case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4056         case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4057         }
4058         return ("AC_UNKNOWN");
4059 }
4060
4061 static int
4062 xpt_async_size(u_int32_t async_code)
4063 {
4064
4065         switch (async_code) {
4066         case AC_BUS_RESET: return (0);
4067         case AC_UNSOL_RESEL: return (0);
4068         case AC_SCSI_AEN: return (0);
4069         case AC_SENT_BDR: return (0);
4070         case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4071         case AC_PATH_DEREGISTERED: return (0);
4072         case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4073         case AC_LOST_DEVICE: return (0);
4074         case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4075         case AC_INQ_CHANGED: return (0);
4076         case AC_GETDEV_CHANGED: return (0);
4077         case AC_CONTRACT: return (sizeof(struct ac_contract));
4078         case AC_ADVINFO_CHANGED: return (-1);
4079         case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4080         }
4081         return (0);
4082 }
4083
4084 static int
4085 xpt_async_process_dev(struct cam_ed *device, void *arg)
4086 {
4087         union ccb *ccb = arg;
4088         struct cam_path *path = ccb->ccb_h.path;
4089         void *async_arg = ccb->casync.async_arg_ptr;
4090         u_int32_t async_code = ccb->casync.async_code;
4091         int relock;
4092
4093         if (path->device != device
4094          && path->device->lun_id != CAM_LUN_WILDCARD
4095          && device->lun_id != CAM_LUN_WILDCARD)
4096                 return (1);
4097
4098         /*
4099          * The async callback could free the device.
4100          * If it is a broadcast async, it doesn't hold
4101          * device reference, so take our own reference.
4102          */
4103         xpt_acquire_device(device);
4104
4105         /*
4106          * If async for specific device is to be delivered to
4107          * the wildcard client, take the specific device lock.
4108          * XXX: We may need a way for client to specify it.
4109          */
4110         if ((device->lun_id == CAM_LUN_WILDCARD &&
4111              path->device->lun_id != CAM_LUN_WILDCARD) ||
4112             (device->target->target_id == CAM_TARGET_WILDCARD &&
4113              path->target->target_id != CAM_TARGET_WILDCARD) ||
4114             (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4115              path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4116                 mtx_unlock(&device->device_mtx);
4117                 xpt_path_lock(path);
4118                 relock = 1;
4119         } else
4120                 relock = 0;
4121
4122         (*(device->target->bus->xport->async))(async_code,
4123             device->target->bus, device->target, device, async_arg);
4124         xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4125
4126         if (relock) {
4127                 xpt_path_unlock(path);
4128                 mtx_lock(&device->device_mtx);
4129         }
4130         xpt_release_device(device);
4131         return (1);
4132 }
4133
4134 static int
4135 xpt_async_process_tgt(struct cam_et *target, void *arg)
4136 {
4137         union ccb *ccb = arg;
4138         struct cam_path *path = ccb->ccb_h.path;
4139
4140         if (path->target != target
4141          && path->target->target_id != CAM_TARGET_WILDCARD
4142          && target->target_id != CAM_TARGET_WILDCARD)
4143                 return (1);
4144
4145         if (ccb->casync.async_code == AC_SENT_BDR) {
4146                 /* Update our notion of when the last reset occurred */
4147                 microtime(&target->last_reset);
4148         }
4149
4150         return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4151 }
4152
4153 static void
4154 xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4155 {
4156         struct cam_eb *bus;
4157         struct cam_path *path;
4158         void *async_arg;
4159         u_int32_t async_code;
4160
4161         path = ccb->ccb_h.path;
4162         async_code = ccb->casync.async_code;
4163         async_arg = ccb->casync.async_arg_ptr;
4164         CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4165             ("xpt_async(%s)\n", xpt_async_string(async_code)));
4166         bus = path->bus;
4167
4168         if (async_code == AC_BUS_RESET) {
4169                 /* Update our notion of when the last reset occurred */
4170                 microtime(&bus->last_reset);
4171         }
4172
4173         xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4174
4175         /*
4176          * If this wasn't a fully wildcarded async, tell all
4177          * clients that want all async events.
4178          */
4179         if (bus != xpt_periph->path->bus) {
4180                 xpt_path_lock(xpt_periph->path);
4181                 xpt_async_process_dev(xpt_periph->path->device, ccb);
4182                 xpt_path_unlock(xpt_periph->path);
4183         }
4184
4185         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4186                 xpt_release_devq(path, 1, TRUE);
4187         else
4188                 xpt_release_simq(path->bus->sim, TRUE);
4189         if (ccb->casync.async_arg_size > 0)
4190                 free(async_arg, M_CAMXPT);
4191         xpt_free_path(path);
4192         xpt_free_ccb(ccb);
4193 }
4194
4195 static void
4196 xpt_async_bcast(struct async_list *async_head,
4197                 u_int32_t async_code,
4198                 struct cam_path *path, void *async_arg)
4199 {
4200         struct async_node *cur_entry;
4201         int lock;
4202
4203         cur_entry = SLIST_FIRST(async_head);
4204         while (cur_entry != NULL) {
4205                 struct async_node *next_entry;
4206                 /*
4207                  * Grab the next list entry before we call the current
4208                  * entry's callback.  This is because the callback function
4209                  * can delete its async callback entry.
4210                  */
4211                 next_entry = SLIST_NEXT(cur_entry, links);
4212                 if ((cur_entry->event_enable & async_code) != 0) {
4213                         lock = cur_entry->event_lock;
4214                         if (lock)
4215                                 CAM_SIM_LOCK(path->device->sim);
4216                         cur_entry->callback(cur_entry->callback_arg,
4217                                             async_code, path,
4218                                             async_arg);
4219                         if (lock)
4220                                 CAM_SIM_UNLOCK(path->device->sim);
4221                 }
4222                 cur_entry = next_entry;
4223         }
4224 }
4225
4226 void
4227 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4228 {
4229         union ccb *ccb;
4230         int size;
4231
4232         ccb = xpt_alloc_ccb_nowait();
4233         if (ccb == NULL) {
4234                 xpt_print(path, "Can't allocate CCB to send %s\n",
4235                     xpt_async_string(async_code));
4236                 return;
4237         }
4238
4239         if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4240                 xpt_print(path, "Can't allocate path to send %s\n",
4241                     xpt_async_string(async_code));
4242                 xpt_free_ccb(ccb);
4243                 return;
4244         }
4245         ccb->ccb_h.path->periph = NULL;
4246         ccb->ccb_h.func_code = XPT_ASYNC;
4247         ccb->ccb_h.cbfcnp = xpt_async_process;
4248         ccb->ccb_h.flags |= CAM_UNLOCKED;
4249         ccb->casync.async_code = async_code;
4250         ccb->casync.async_arg_size = 0;
4251         size = xpt_async_size(async_code);
4252         if (size > 0 && async_arg != NULL) {
4253                 ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4254                 if (ccb->casync.async_arg_ptr == NULL) {
4255                         xpt_print(path, "Can't allocate argument to send %s\n",
4256                             xpt_async_string(async_code));
4257                         xpt_free_path(ccb->ccb_h.path);
4258                         xpt_free_ccb(ccb);
4259                         return;
4260                 }
4261                 memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4262                 ccb->casync.async_arg_size = size;
4263         } else if (size < 0)
4264                 ccb->casync.async_arg_size = size;
4265         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4266                 xpt_freeze_devq(path, 1);
4267         else
4268                 xpt_freeze_simq(path->bus->sim, 1);
4269         xpt_done(ccb);
4270 }
4271
4272 static void
4273 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4274                       struct cam_et *target, struct cam_ed *device,
4275                       void *async_arg)
4276 {
4277
4278         /*
4279          * We only need to handle events for real devices.
4280          */
4281         if (target->target_id == CAM_TARGET_WILDCARD
4282          || device->lun_id == CAM_LUN_WILDCARD)
4283                 return;
4284
4285         printf("%s called\n", __func__);
4286 }
4287
4288 u_int32_t
4289 xpt_freeze_devq(struct cam_path *path, u_int count)
4290 {
4291         struct cam_ed   *dev = path->device;
4292         struct cam_devq *devq;
4293         uint32_t         freeze;
4294
4295         devq = dev->sim->devq;
4296         mtx_lock(&devq->send_mtx);
4297         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq() %u->%u\n",
4298             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4299         freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4300         /* Remove frozen device from sendq. */
4301         if (device_is_queued(dev))
4302                 camq_remove(&devq->send_queue, dev->devq_entry.index);
4303         mtx_unlock(&devq->send_mtx);
4304         return (freeze);
4305 }
4306
4307 u_int32_t
4308 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4309 {
4310         struct cam_devq *devq;
4311         uint32_t         freeze;
4312
4313         devq = sim->devq;
4314         mtx_lock(&devq->send_mtx);
4315         freeze = (devq->send_queue.qfrozen_cnt += count);
4316         mtx_unlock(&devq->send_mtx);
4317         return (freeze);
4318 }
4319
4320 static void
4321 xpt_release_devq_timeout(void *arg)
4322 {
4323         struct cam_ed *dev;
4324         struct cam_devq *devq;
4325
4326         dev = (struct cam_ed *)arg;
4327         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4328         devq = dev->sim->devq;
4329         mtx_assert(&devq->send_mtx, MA_OWNED);
4330         if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4331                 xpt_run_devq(devq);
4332 }
4333
4334 void
4335 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4336 {
4337         struct cam_ed *dev;
4338         struct cam_devq *devq;
4339
4340         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4341             count, run_queue));
4342         dev = path->device;
4343         devq = dev->sim->devq;
4344         mtx_lock(&devq->send_mtx);
4345         if (xpt_release_devq_device(dev, count, run_queue))
4346                 xpt_run_devq(dev->sim->devq);
4347         mtx_unlock(&devq->send_mtx);
4348 }
4349
4350 static int
4351 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4352 {
4353
4354         mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4355         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4356             ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4357             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4358         if (count > dev->ccbq.queue.qfrozen_cnt) {
4359 #ifdef INVARIANTS
4360                 printf("xpt_release_devq(): requested %u > present %u\n",
4361                     count, dev->ccbq.queue.qfrozen_cnt);
4362 #endif
4363                 count = dev->ccbq.queue.qfrozen_cnt;
4364         }
4365         dev->ccbq.queue.qfrozen_cnt -= count;
4366         if (dev->ccbq.queue.qfrozen_cnt == 0) {
4367                 /*
4368                  * No longer need to wait for a successful
4369                  * command completion.
4370                  */
4371                 dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4372                 /*
4373                  * Remove any timeouts that might be scheduled
4374                  * to release this queue.
4375                  */
4376                 if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4377                         callout_stop(&dev->callout);
4378                         dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4379                 }
4380                 /*
4381                  * Now that we are unfrozen schedule the
4382                  * device so any pending transactions are
4383                  * run.
4384                  */
4385                 xpt_schedule_devq(dev->sim->devq, dev);
4386         } else
4387                 run_queue = 0;
4388         return (run_queue);
4389 }
4390
4391 void
4392 xpt_release_simq(struct cam_sim *sim, int run_queue)
4393 {
4394         struct cam_devq *devq;
4395
4396         devq = sim->devq;
4397         mtx_lock(&devq->send_mtx);
4398         if (devq->send_queue.qfrozen_cnt <= 0) {
4399 #ifdef INVARIANTS
4400                 printf("xpt_release_simq: requested 1 > present %u\n",
4401                     devq->send_queue.qfrozen_cnt);
4402 #endif
4403         } else
4404                 devq->send_queue.qfrozen_cnt--;
4405         if (devq->send_queue.qfrozen_cnt == 0) {
4406                 /*
4407                  * If there is a timeout scheduled to release this
4408                  * sim queue, remove it.  The queue frozen count is
4409                  * already at 0.
4410                  */
4411                 if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4412                         callout_stop(&sim->callout);
4413                         sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4414                 }
4415                 if (run_queue) {
4416                         /*
4417                          * Now that we are unfrozen run the send queue.
4418                          */
4419                         xpt_run_devq(sim->devq);
4420                 }
4421         }
4422         mtx_unlock(&devq->send_mtx);
4423 }
4424
4425 /*
4426  * XXX Appears to be unused.
4427  */
4428 static void
4429 xpt_release_simq_timeout(void *arg)
4430 {
4431         struct cam_sim *sim;
4432
4433         sim = (struct cam_sim *)arg;
4434         xpt_release_simq(sim, /* run_queue */ TRUE);
4435 }
4436
4437 void
4438 xpt_done(union ccb *done_ccb)
4439 {
4440         struct cam_doneq *queue;
4441         int     run, hash;
4442
4443         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4444         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4445                 return;
4446
4447         hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4448             done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4449         queue = &cam_doneqs[hash];
4450         mtx_lock(&queue->cam_doneq_mtx);
4451         run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4452         STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4453         done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4454         mtx_unlock(&queue->cam_doneq_mtx);
4455         if (run)
4456                 wakeup(&queue->cam_doneq);
4457 }
4458
4459 void
4460 xpt_done_direct(union ccb *done_ccb)
4461 {
4462
4463         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done_direct\n"));
4464         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4465                 return;
4466
4467         xpt_done_process(&done_ccb->ccb_h);
4468 }
4469
4470 union ccb *
4471 xpt_alloc_ccb()
4472 {
4473         union ccb *new_ccb;
4474
4475         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4476         return (new_ccb);
4477 }
4478
4479 union ccb *
4480 xpt_alloc_ccb_nowait()
4481 {
4482         union ccb *new_ccb;
4483
4484         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4485         return (new_ccb);
4486 }
4487
4488 void
4489 xpt_free_ccb(union ccb *free_ccb)
4490 {
4491         free(free_ccb, M_CAMCCB);
4492 }
4493
4494
4495
4496 /* Private XPT functions */
4497
4498 /*
4499  * Get a CAM control block for the caller. Charge the structure to the device
4500  * referenced by the path.  If we don't have sufficient resources to allocate
4501  * more ccbs, we return NULL.
4502  */
4503 static union ccb *
4504 xpt_get_ccb_nowait(struct cam_periph *periph)
4505 {
4506         union ccb *new_ccb;
4507
4508         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_NOWAIT);
4509         if (new_ccb == NULL)
4510                 return (NULL);
4511         periph->periph_allocated++;
4512         cam_ccbq_take_opening(&periph->path->device->ccbq);
4513         return (new_ccb);
4514 }
4515
4516 static union ccb *
4517 xpt_get_ccb(struct cam_periph *periph)
4518 {
4519         union ccb *new_ccb;
4520
4521         cam_periph_unlock(periph);
4522         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_WAITOK);
4523         cam_periph_lock(periph);
4524         periph->periph_allocated++;
4525         cam_ccbq_take_opening(&periph->path->device->ccbq);
4526         return (new_ccb);
4527 }
4528
4529 union ccb *
4530 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4531 {
4532         struct ccb_hdr *ccb_h;
4533
4534         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4535         cam_periph_assert(periph, MA_OWNED);
4536         while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4537             ccb_h->pinfo.priority != priority) {
4538                 if (priority < periph->immediate_priority) {
4539                         periph->immediate_priority = priority;
4540                         xpt_run_allocq(periph, 0);
4541                 } else
4542                         cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4543                             "cgticb", 0);
4544         }
4545         SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4546         return ((union ccb *)ccb_h);
4547 }
4548
4549 static void
4550 xpt_acquire_bus(struct cam_eb *bus)
4551 {
4552
4553         xpt_lock_buses();
4554         bus->refcount++;
4555         xpt_unlock_buses();
4556 }
4557
4558 static void
4559 xpt_release_bus(struct cam_eb *bus)
4560 {
4561
4562         xpt_lock_buses();
4563         KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4564         if (--bus->refcount > 0) {
4565                 xpt_unlock_buses();
4566                 return;
4567         }
4568         TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4569         xsoftc.bus_generation++;
4570         xpt_unlock_buses();
4571         KASSERT(TAILQ_EMPTY(&bus->et_entries),
4572             ("destroying bus, but target list is not empty"));
4573         cam_sim_release(bus->sim);
4574         mtx_destroy(&bus->eb_mtx);
4575         free(bus, M_CAMXPT);
4576 }
4577
4578 static struct cam_et *
4579 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4580 {
4581         struct cam_et *cur_target, *target;
4582
4583         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4584         mtx_assert(&bus->eb_mtx, MA_OWNED);
4585         target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4586                                          M_NOWAIT|M_ZERO);
4587         if (target == NULL)
4588                 return (NULL);
4589
4590         TAILQ_INIT(&target->ed_entries);
4591         target->bus = bus;
4592         target->target_id = target_id;
4593         target->refcount = 1;
4594         target->generation = 0;
4595         target->luns = NULL;
4596         mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4597         timevalclear(&target->last_reset);
4598         /*
4599          * Hold a reference to our parent bus so it
4600          * will not go away before we do.
4601          */
4602         bus->refcount++;
4603
4604         /* Insertion sort into our bus's target list */
4605         cur_target = TAILQ_FIRST(&bus->et_entries);
4606         while (cur_target != NULL && cur_target->target_id < target_id)
4607                 cur_target = TAILQ_NEXT(cur_target, links);
4608         if (cur_target != NULL) {
4609                 TAILQ_INSERT_BEFORE(cur_target, target, links);
4610         } else {
4611                 TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4612         }
4613         bus->generation++;
4614         return (target);
4615 }
4616
4617 static void
4618 xpt_acquire_target(struct cam_et *target)
4619 {
4620         struct cam_eb *bus = target->bus;
4621
4622         mtx_lock(&bus->eb_mtx);
4623         target->refcount++;
4624         mtx_unlock(&bus->eb_mtx);
4625 }
4626
4627 static void
4628 xpt_release_target(struct cam_et *target)
4629 {
4630         struct cam_eb *bus = target->bus;
4631
4632         mtx_lock(&bus->eb_mtx);
4633         if (--target->refcount > 0) {
4634                 mtx_unlock(&bus->eb_mtx);
4635                 return;
4636         }
4637         TAILQ_REMOVE(&bus->et_entries, target, links);
4638         bus->generation++;
4639         mtx_unlock(&bus->eb_mtx);
4640         KASSERT(TAILQ_EMPTY(&target->ed_entries),
4641             ("destroying target, but device list is not empty"));
4642         xpt_release_bus(bus);
4643         mtx_destroy(&target->luns_mtx);
4644         if (target->luns)
4645                 free(target->luns, M_CAMXPT);
4646         free(target, M_CAMXPT);
4647 }
4648
4649 static struct cam_ed *
4650 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4651                          lun_id_t lun_id)
4652 {
4653         struct cam_ed *device;
4654
4655         device = xpt_alloc_device(bus, target, lun_id);
4656         if (device == NULL)
4657                 return (NULL);
4658
4659         device->mintags = 1;
4660         device->maxtags = 1;
4661         return (device);
4662 }
4663
4664 static void
4665 xpt_destroy_device(void *context, int pending)
4666 {
4667         struct cam_ed   *device = context;
4668
4669         mtx_lock(&device->device_mtx);
4670         mtx_destroy(&device->device_mtx);
4671         free(device, M_CAMDEV);
4672 }
4673
4674 struct cam_ed *
4675 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4676 {
4677         struct cam_ed   *cur_device, *device;
4678         struct cam_devq *devq;
4679         cam_status status;
4680
4681         mtx_assert(&bus->eb_mtx, MA_OWNED);
4682         /* Make space for us in the device queue on our bus */
4683         devq = bus->sim->devq;
4684         mtx_lock(&devq->send_mtx);
4685         status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4686         mtx_unlock(&devq->send_mtx);
4687         if (status != CAM_REQ_CMP)
4688                 return (NULL);
4689
4690         device = (struct cam_ed *)malloc(sizeof(*device),
4691                                          M_CAMDEV, M_NOWAIT|M_ZERO);
4692         if (device == NULL)
4693                 return (NULL);
4694
4695         cam_init_pinfo(&device->devq_entry);
4696         device->target = target;
4697         device->lun_id = lun_id;
4698         device->sim = bus->sim;
4699         if (cam_ccbq_init(&device->ccbq,
4700                           bus->sim->max_dev_openings) != 0) {
4701                 free(device, M_CAMDEV);
4702                 return (NULL);
4703         }
4704         SLIST_INIT(&device->asyncs);
4705         SLIST_INIT(&device->periphs);
4706         device->generation = 0;
4707         device->flags = CAM_DEV_UNCONFIGURED;
4708         device->tag_delay_count = 0;
4709         device->tag_saved_openings = 0;
4710         device->refcount = 1;
4711         mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4712         callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4713         TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4714         /*
4715          * Hold a reference to our parent bus so it
4716          * will not go away before we do.
4717          */
4718         target->refcount++;
4719
4720         cur_device = TAILQ_FIRST(&target->ed_entries);
4721         while (cur_device != NULL && cur_device->lun_id < lun_id)
4722                 cur_device = TAILQ_NEXT(cur_device, links);
4723         if (cur_device != NULL)
4724                 TAILQ_INSERT_BEFORE(cur_device, device, links);
4725         else
4726                 TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4727         target->generation++;
4728         return (device);
4729 }
4730
4731 void
4732 xpt_acquire_device(struct cam_ed *device)
4733 {
4734         struct cam_eb *bus = device->target->bus;
4735
4736         mtx_lock(&bus->eb_mtx);
4737         device->refcount++;
4738         mtx_unlock(&bus->eb_mtx);
4739 }
4740
4741 void
4742 xpt_release_device(struct cam_ed *device)
4743 {
4744         struct cam_eb *bus = device->target->bus;
4745         struct cam_devq *devq;
4746
4747         mtx_lock(&bus->eb_mtx);
4748         if (--device->refcount > 0) {
4749                 mtx_unlock(&bus->eb_mtx);
4750                 return;
4751         }
4752
4753         TAILQ_REMOVE(&device->target->ed_entries, device,links);
4754         device->target->generation++;
4755         mtx_unlock(&bus->eb_mtx);
4756
4757         /* Release our slot in the devq */
4758         devq = bus->sim->devq;
4759         mtx_lock(&devq->send_mtx);
4760         cam_devq_resize(devq, devq->send_queue.array_size - 1);
4761         mtx_unlock(&devq->send_mtx);
4762
4763         KASSERT(SLIST_EMPTY(&device->periphs),
4764             ("destroying device, but periphs list is not empty"));
4765         KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4766             ("destroying device while still queued for ccbs"));
4767
4768         if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4769                 callout_stop(&device->callout);
4770
4771         xpt_release_target(device->target);
4772
4773         cam_ccbq_fini(&device->ccbq);
4774         /*
4775          * Free allocated memory.  free(9) does nothing if the
4776          * supplied pointer is NULL, so it is safe to call without
4777          * checking.
4778          */
4779         free(device->supported_vpds, M_CAMXPT);
4780         free(device->device_id, M_CAMXPT);
4781         free(device->physpath, M_CAMXPT);
4782         free(device->rcap_buf, M_CAMXPT);
4783         free(device->serial_num, M_CAMXPT);
4784         taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4785 }
4786
4787 u_int32_t
4788 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4789 {
4790         int     result;
4791         struct  cam_ed *dev;
4792
4793         dev = path->device;
4794         mtx_lock(&dev->sim->devq->send_mtx);
4795         result = cam_ccbq_resize(&dev->ccbq, newopenings);
4796         mtx_unlock(&dev->sim->devq->send_mtx);
4797         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4798          || (dev->inq_flags & SID_CmdQue) != 0)
4799                 dev->tag_saved_openings = newopenings;
4800         return (result);
4801 }
4802
4803 static struct cam_eb *
4804 xpt_find_bus(path_id_t path_id)
4805 {
4806         struct cam_eb *bus;
4807
4808         xpt_lock_buses();
4809         for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4810              bus != NULL;
4811              bus = TAILQ_NEXT(bus, links)) {
4812                 if (bus->path_id == path_id) {
4813                         bus->refcount++;
4814                         break;
4815                 }
4816         }
4817         xpt_unlock_buses();
4818         return (bus);
4819 }
4820
4821 static struct cam_et *
4822 xpt_find_target(struct cam_eb *bus, target_id_t target_id)
4823 {
4824         struct cam_et *target;
4825
4826         mtx_assert(&bus->eb_mtx, MA_OWNED);
4827         for (target = TAILQ_FIRST(&bus->et_entries);
4828              target != NULL;
4829              target = TAILQ_NEXT(target, links)) {
4830                 if (target->target_id == target_id) {
4831                         target->refcount++;
4832                         break;
4833                 }
4834         }
4835         return (target);
4836 }
4837
4838 static struct cam_ed *
4839 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4840 {
4841         struct cam_ed *device;
4842
4843         mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4844         for (device = TAILQ_FIRST(&target->ed_entries);
4845              device != NULL;
4846              device = TAILQ_NEXT(device, links)) {
4847                 if (device->lun_id == lun_id) {
4848                         device->refcount++;
4849                         break;
4850                 }
4851         }
4852         return (device);
4853 }
4854
4855 void
4856 xpt_start_tags(struct cam_path *path)
4857 {
4858         struct ccb_relsim crs;
4859         struct cam_ed *device;
4860         struct cam_sim *sim;
4861         int    newopenings;
4862
4863         device = path->device;
4864         sim = path->bus->sim;
4865         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4866         xpt_freeze_devq(path, /*count*/1);
4867         device->inq_flags |= SID_CmdQue;
4868         if (device->tag_saved_openings != 0)
4869                 newopenings = device->tag_saved_openings;
4870         else
4871                 newopenings = min(device->maxtags,
4872                                   sim->max_tagged_dev_openings);
4873         xpt_dev_ccbq_resize(path, newopenings);
4874         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4875         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4876         crs.ccb_h.func_code = XPT_REL_SIMQ;
4877         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4878         crs.openings
4879             = crs.release_timeout
4880             = crs.qfrozen_cnt
4881             = 0;
4882         xpt_action((union ccb *)&crs);
4883 }
4884
4885 void
4886 xpt_stop_tags(struct cam_path *path)
4887 {
4888         struct ccb_relsim crs;
4889         struct cam_ed *device;
4890         struct cam_sim *sim;
4891
4892         device = path->device;
4893         sim = path->bus->sim;
4894         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4895         device->tag_delay_count = 0;
4896         xpt_freeze_devq(path, /*count*/1);
4897         device->inq_flags &= ~SID_CmdQue;
4898         xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4899         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4900         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4901         crs.ccb_h.func_code = XPT_REL_SIMQ;
4902         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4903         crs.openings
4904             = crs.release_timeout
4905             = crs.qfrozen_cnt
4906             = 0;
4907         xpt_action((union ccb *)&crs);
4908 }
4909
4910 static void
4911 xpt_boot_delay(void *arg)
4912 {
4913
4914         xpt_release_boot();
4915 }
4916
4917 static void
4918 xpt_config(void *arg)
4919 {
4920         /*
4921          * Now that interrupts are enabled, go find our devices
4922          */
4923         if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
4924                 printf("xpt_config: failed to create taskqueue thread.\n");
4925
4926         /* Setup debugging path */
4927         if (cam_dflags != CAM_DEBUG_NONE) {
4928                 if (xpt_create_path(&cam_dpath, NULL,
4929                                     CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
4930                                     CAM_DEBUG_LUN) != CAM_REQ_CMP) {
4931                         printf("xpt_config: xpt_create_path() failed for debug"
4932                                " target %d:%d:%d, debugging disabled\n",
4933                                CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
4934                         cam_dflags = CAM_DEBUG_NONE;
4935                 }
4936         } else
4937                 cam_dpath = NULL;
4938
4939         periphdriver_init(1);
4940         xpt_hold_boot();
4941         callout_init(&xsoftc.boot_callout, 1);
4942         callout_reset(&xsoftc.boot_callout, hz * xsoftc.boot_delay / 1000,
4943             xpt_boot_delay, NULL);
4944         /* Fire up rescan thread. */
4945         if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
4946             "cam", "scanner")) {
4947                 printf("xpt_config: failed to create rescan thread.\n");
4948         }
4949 }
4950
4951 void
4952 xpt_hold_boot(void)
4953 {
4954         xpt_lock_buses();
4955         xsoftc.buses_to_config++;
4956         xpt_unlock_buses();
4957 }
4958
4959 void
4960 xpt_release_boot(void)
4961 {
4962         xpt_lock_buses();
4963         xsoftc.buses_to_config--;
4964         if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
4965                 struct  xpt_task *task;
4966
4967                 xsoftc.buses_config_done = 1;
4968                 xpt_unlock_buses();
4969                 /* Call manually because we don't have any busses */
4970                 task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
4971                 if (task != NULL) {
4972                         TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
4973                         taskqueue_enqueue(taskqueue_thread, &task->task);
4974                 }
4975         } else
4976                 xpt_unlock_buses();
4977 }
4978
4979 /*
4980  * If the given device only has one peripheral attached to it, and if that
4981  * peripheral is the passthrough driver, announce it.  This insures that the
4982  * user sees some sort of announcement for every peripheral in their system.
4983  */
4984 static int
4985 xptpassannouncefunc(struct cam_ed *device, void *arg)
4986 {
4987         struct cam_periph *periph;
4988         int i;
4989
4990         for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
4991              periph = SLIST_NEXT(periph, periph_links), i++);
4992
4993         periph = SLIST_FIRST(&device->periphs);
4994         if ((i == 1)
4995          && (strncmp(periph->periph_name, "pass", 4) == 0))
4996                 xpt_announce_periph(periph, NULL);
4997
4998         return(1);
4999 }
5000
5001 static void
5002 xpt_finishconfig_task(void *context, int pending)
5003 {
5004
5005         periphdriver_init(2);
5006         /*
5007          * Check for devices with no "standard" peripheral driver
5008          * attached.  For any devices like that, announce the
5009          * passthrough driver so the user will see something.
5010          */
5011         if (!bootverbose)
5012                 xpt_for_all_devices(xptpassannouncefunc, NULL);
5013
5014         /* Release our hook so that the boot can continue. */
5015         config_intrhook_disestablish(xsoftc.xpt_config_hook);
5016         free(xsoftc.xpt_config_hook, M_CAMXPT);
5017         xsoftc.xpt_config_hook = NULL;
5018
5019         free(context, M_CAMXPT);
5020 }
5021
5022 cam_status
5023 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5024                    struct cam_path *path)
5025 {
5026         struct ccb_setasync csa;
5027         cam_status status;
5028         int xptpath = 0;
5029
5030         if (path == NULL) {
5031                 status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5032                                          CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5033                 if (status != CAM_REQ_CMP)
5034                         return (status);
5035                 xpt_path_lock(path);
5036                 xptpath = 1;
5037         }
5038
5039         xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5040         csa.ccb_h.func_code = XPT_SASYNC_CB;
5041         csa.event_enable = event;
5042         csa.callback = cbfunc;
5043         csa.callback_arg = cbarg;
5044         xpt_action((union ccb *)&csa);
5045         status = csa.ccb_h.status;
5046
5047         if (xptpath) {
5048                 xpt_path_unlock(path);
5049                 xpt_free_path(path);
5050         }
5051
5052         if ((status == CAM_REQ_CMP) &&
5053             (csa.event_enable & AC_FOUND_DEVICE)) {
5054                 /*
5055                  * Get this peripheral up to date with all
5056                  * the currently existing devices.
5057                  */
5058                 xpt_for_all_devices(xptsetasyncfunc, &csa);
5059         }
5060         if ((status == CAM_REQ_CMP) &&
5061             (csa.event_enable & AC_PATH_REGISTERED)) {
5062                 /*
5063                  * Get this peripheral up to date with all
5064                  * the currently existing busses.
5065                  */
5066                 xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5067         }
5068
5069         return (status);
5070 }
5071
5072 static void
5073 xptaction(struct cam_sim *sim, union ccb *work_ccb)
5074 {
5075         CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5076
5077         switch (work_ccb->ccb_h.func_code) {
5078         /* Common cases first */
5079         case XPT_PATH_INQ:              /* Path routing inquiry */
5080         {
5081                 struct ccb_pathinq *cpi;
5082
5083                 cpi = &work_ccb->cpi;
5084                 cpi->version_num = 1; /* XXX??? */
5085                 cpi->hba_inquiry = 0;
5086                 cpi->target_sprt = 0;
5087                 cpi->hba_misc = 0;
5088                 cpi->hba_eng_cnt = 0;
5089                 cpi->max_target = 0;
5090                 cpi->max_lun = 0;
5091                 cpi->initiator_id = 0;
5092                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5093                 strncpy(cpi->hba_vid, "", HBA_IDLEN);
5094                 strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5095                 cpi->unit_number = sim->unit_number;
5096                 cpi->bus_id = sim->bus_id;
5097                 cpi->base_transfer_speed = 0;
5098                 cpi->protocol = PROTO_UNSPECIFIED;
5099                 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5100                 cpi->transport = XPORT_UNSPECIFIED;
5101                 cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5102                 cpi->ccb_h.status = CAM_REQ_CMP;
5103                 xpt_done(work_ccb);
5104                 break;
5105         }
5106         default:
5107                 work_ccb->ccb_h.status = CAM_REQ_INVALID;
5108                 xpt_done(work_ccb);
5109                 break;
5110         }
5111 }
5112
5113 /*
5114  * The xpt as a "controller" has no interrupt sources, so polling
5115  * is a no-op.
5116  */
5117 static void
5118 xptpoll(struct cam_sim *sim)
5119 {
5120 }
5121
5122 void
5123 xpt_lock_buses(void)
5124 {
5125         mtx_lock(&xsoftc.xpt_topo_lock);
5126 }
5127
5128 void
5129 xpt_unlock_buses(void)
5130 {
5131         mtx_unlock(&xsoftc.xpt_topo_lock);
5132 }
5133
5134 struct mtx *
5135 xpt_path_mtx(struct cam_path *path)
5136 {
5137
5138         return (&path->device->device_mtx);
5139 }
5140
5141 static void
5142 xpt_done_process(struct ccb_hdr *ccb_h)
5143 {
5144         struct cam_sim *sim;
5145         struct cam_devq *devq;
5146         struct mtx *mtx = NULL;
5147
5148         if (ccb_h->flags & CAM_HIGH_POWER) {
5149                 struct highpowerlist    *hphead;
5150                 struct cam_ed           *device;
5151
5152                 mtx_lock(&xsoftc.xpt_lock);
5153                 hphead = &xsoftc.highpowerq;
5154
5155                 device = STAILQ_FIRST(hphead);
5156
5157                 /*
5158                  * Increment the count since this command is done.
5159                  */
5160                 xsoftc.num_highpower++;
5161
5162                 /*
5163                  * Any high powered commands queued up?
5164                  */
5165                 if (device != NULL) {
5166
5167                         STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5168                         mtx_unlock(&xsoftc.xpt_lock);
5169
5170                         mtx_lock(&device->sim->devq->send_mtx);
5171                         xpt_release_devq_device(device,
5172                                          /*count*/1, /*runqueue*/TRUE);
5173                         mtx_unlock(&device->sim->devq->send_mtx);
5174                 } else
5175                         mtx_unlock(&xsoftc.xpt_lock);
5176         }
5177
5178         sim = ccb_h->path->bus->sim;
5179
5180         if (ccb_h->status & CAM_RELEASE_SIMQ) {
5181                 xpt_release_simq(sim, /*run_queue*/FALSE);
5182                 ccb_h->status &= ~CAM_RELEASE_SIMQ;
5183         }
5184
5185         if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5186          && (ccb_h->status & CAM_DEV_QFRZN)) {
5187                 xpt_release_devq(ccb_h->path, /*count*/1,
5188                                  /*run_queue*/FALSE);
5189                 ccb_h->status &= ~CAM_DEV_QFRZN;
5190         }
5191
5192         devq = sim->devq;
5193         if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5194                 struct cam_ed *dev = ccb_h->path->device;
5195
5196                 mtx_lock(&devq->send_mtx);
5197                 devq->send_active--;
5198                 devq->send_openings++;
5199                 cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5200
5201                 if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5202                   && (dev->ccbq.dev_active == 0))) {
5203                         dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5204                         xpt_release_devq_device(dev, /*count*/1,
5205                                          /*run_queue*/FALSE);
5206                 }
5207
5208                 if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5209                   && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5210                         dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5211                         xpt_release_devq_device(dev, /*count*/1,
5212                                          /*run_queue*/FALSE);
5213                 }
5214
5215                 if (!device_is_queued(dev))
5216                         (void)xpt_schedule_devq(devq, dev);
5217                 mtx_unlock(&devq->send_mtx);
5218
5219                 if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5220                         mtx = xpt_path_mtx(ccb_h->path);
5221                         mtx_lock(mtx);
5222
5223                         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5224                          && (--dev->tag_delay_count == 0))
5225                                 xpt_start_tags(ccb_h->path);
5226                 }
5227         }
5228
5229         if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5230                 if (mtx == NULL) {
5231                         mtx = xpt_path_mtx(ccb_h->path);
5232                         mtx_lock(mtx);
5233                 }
5234         } else {
5235                 if (mtx != NULL) {
5236                         mtx_unlock(mtx);
5237                         mtx = NULL;
5238                 }
5239         }
5240
5241         /* Call the peripheral driver's callback */
5242         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5243         (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5244         if (mtx != NULL)
5245                 mtx_unlock(mtx);
5246
5247         mtx_lock(&devq->send_mtx);
5248         xpt_run_devq(devq);
5249         mtx_unlock(&devq->send_mtx);
5250 }
5251
5252 void
5253 xpt_done_td(void *arg)
5254 {
5255         struct cam_doneq *queue = arg;
5256         struct ccb_hdr *ccb_h;
5257         STAILQ_HEAD(, ccb_hdr)  doneq;
5258
5259         STAILQ_INIT(&doneq);
5260         mtx_lock(&queue->cam_doneq_mtx);
5261         while (1) {
5262                 while (STAILQ_EMPTY(&queue->cam_doneq)) {
5263                         queue->cam_doneq_sleep = 1;
5264                         msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5265                             PRIBIO, "-", 0);
5266                         queue->cam_doneq_sleep = 0;
5267                 }
5268                 STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5269                 mtx_unlock(&queue->cam_doneq_mtx);
5270
5271                 THREAD_NO_SLEEPING();
5272                 while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5273                         STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5274                         xpt_done_process(ccb_h);
5275                 }
5276                 THREAD_SLEEPING_OK();
5277
5278                 mtx_lock(&queue->cam_doneq_mtx);
5279         }
5280 }
5281
5282 static void
5283 camisr_runqueue(void)
5284 {
5285         struct  ccb_hdr *ccb_h;
5286         struct cam_doneq *queue;
5287         int i;
5288
5289         /* Process global queues. */
5290         for (i = 0; i < cam_num_doneqs; i++) {
5291                 queue = &cam_doneqs[i];
5292                 mtx_lock(&queue->cam_doneq_mtx);
5293                 while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5294                         STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5295                         mtx_unlock(&queue->cam_doneq_mtx);
5296                         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5297                         xpt_done_process(ccb_h);
5298                         mtx_lock(&queue->cam_doneq_mtx);
5299                 }
5300                 mtx_unlock(&queue->cam_doneq_mtx);
5301         }
5302 }