sys/cam/cam_xpt.c

   1 /*-
   2  * Implementation of the Common Access Method Transport (XPT) layer.
   3  *
   4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
   5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification, immediately at the beginning of the file.
  14  * 2. The name of the author may not be used to endorse or promote products
  15  *    derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <sys/param.h>
  34 #include <sys/bus.h>
  35 #include <sys/systm.h>
  36 #include <sys/types.h>
  37 #include <sys/malloc.h>
  38 #include <sys/kernel.h>
  39 #include <sys/time.h>
  40 #include <sys/conf.h>
  41 #include <sys/fcntl.h>
  42 #include <sys/interrupt.h>
  43 #include <sys/proc.h>
  44 #include <sys/sbuf.h>
  45 #include <sys/smp.h>
  46 #include <sys/taskqueue.h>
  47
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/sysctl.h>
  51 #include <sys/kthread.h>
  52
  53 #include <cam/cam.h>
  54 #include <cam/cam_ccb.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/cam_queue.h>
  57 #include <cam/cam_sim.h>
  58 #include <cam/cam_xpt.h>
  59 #include <cam/cam_xpt_sim.h>
  60 #include <cam/cam_xpt_periph.h>
  61 #include <cam/cam_xpt_internal.h>
  62 #include <cam/cam_debug.h>
  63 #include <cam/cam_compat.h>
  64
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_message.h>
  67 #include <cam/scsi/scsi_pass.h>
  68
  69 #include <machine/md_var.h>     /* geometry translation */
  70 #include <machine/stdarg.h>     /* for xpt_print below */
  71
  72 #include "opt_cam.h"
  73
  74 /*
  75  * This is the maximum number of high powered commands (e.g. start unit)
  76  * that can be outstanding at a particular time.
  77  */
  78 #ifndef CAM_MAX_HIGHPOWER
  79 #define CAM_MAX_HIGHPOWER  4
  80 #endif
  81
  82 /* Datastructures internal to the xpt layer */
  83 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
  84 MALLOC_DEFINE(M_CAMDEV, "CAM DEV", "CAM devices");
  85 MALLOC_DEFINE(M_CAMCCB, "CAM CCB", "CAM CCBs");
  86 MALLOC_DEFINE(M_CAMPATH, "CAM path", "CAM paths");
  87
  88 /* Object for defering XPT actions to a taskqueue */
  89 struct xpt_task {
  90         struct task     task;
  91         void            *data1;
  92         uintptr_t       data2;
  93 };
  94
  95 struct xpt_softc {
  96         uint32_t                xpt_generation;
  97
  98         /* number of high powered commands that can go through right now */
  99         struct mtx              xpt_highpower_lock;
 100         STAILQ_HEAD(highpowerlist, cam_ed)      highpowerq;
 101         int                     num_highpower;
 102
 103         /* queue for handling async rescan requests. */
 104         TAILQ_HEAD(, ccb_hdr) ccb_scanq;
 105         int buses_to_config;
 106         int buses_config_done;
 107
 108         /* Registered busses */
 109         TAILQ_HEAD(,cam_eb)     xpt_busses;
 110         u_int                   bus_generation;
 111
 112         struct intr_config_hook *xpt_config_hook;
 113
 114         int                     boot_delay;
 115         struct callout          boot_callout;
 116
 117         struct mtx              xpt_topo_lock;
 118         struct mtx              xpt_lock;
 119         struct taskqueue        *xpt_taskq;
 120 };
 121
 122 typedef enum {
 123         DM_RET_COPY             = 0x01,
 124         DM_RET_FLAG_MASK        = 0x0f,
 125         DM_RET_NONE             = 0x00,
 126         DM_RET_STOP             = 0x10,
 127         DM_RET_DESCEND          = 0x20,
 128         DM_RET_ERROR            = 0x30,
 129         DM_RET_ACTION_MASK      = 0xf0
 130 } dev_match_ret;
 131
 132 typedef enum {
 133         XPT_DEPTH_BUS,
 134         XPT_DEPTH_TARGET,
 135         XPT_DEPTH_DEVICE,
 136         XPT_DEPTH_PERIPH
 137 } xpt_traverse_depth;
 138
 139 struct xpt_traverse_config {
 140         xpt_traverse_depth      depth;
 141         void                    *tr_func;
 142         void                    *tr_arg;
 143 };
 144
 145 typedef int     xpt_busfunc_t (struct cam_eb *bus, void *arg);
 146 typedef int     xpt_targetfunc_t (struct cam_et *target, void *arg);
 147 typedef int     xpt_devicefunc_t (struct cam_ed *device, void *arg);
 148 typedef int     xpt_periphfunc_t (struct cam_periph *periph, void *arg);
 149 typedef int     xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
 150
 151 /* Transport layer configuration information */
 152 static struct xpt_softc xsoftc;
 153
 154 MTX_SYSINIT(xpt_topo_init, &xsoftc.xpt_topo_lock, "XPT topology lock", MTX_DEF);
 155
 156 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
 157            &xsoftc.boot_delay, 0, "Bus registration wait time");
 158 SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
 159             &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
 160
 161 struct cam_doneq {
 162         struct mtx_padalign     cam_doneq_mtx;
 163         STAILQ_HEAD(, ccb_hdr)  cam_doneq;
 164         int                     cam_doneq_sleep;
 165 };
 166
 167 static struct cam_doneq cam_doneqs[MAXCPU];
 168 static int cam_num_doneqs;
 169 static struct proc *cam_proc;
 170
 171 SYSCTL_INT(_kern_cam, OID_AUTO, num_doneqs, CTLFLAG_RDTUN,
 172            &cam_num_doneqs, 0, "Number of completion queues/threads");
 173
 174 struct cam_periph *xpt_periph;
 175
 176 static periph_init_t xpt_periph_init;
 177
 178 static struct periph_driver xpt_driver =
 179 {
 180         xpt_periph_init, "xpt",
 181         TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
 182         CAM_PERIPH_DRV_EARLY
 183 };
 184
 185 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
 186
 187 static d_open_t xptopen;
 188 static d_close_t xptclose;
 189 static d_ioctl_t xptioctl;
 190 static d_ioctl_t xptdoioctl;
 191
 192 static struct cdevsw xpt_cdevsw = {
 193         .d_version =    D_VERSION,
 194         .d_flags =      0,
 195         .d_open =       xptopen,
 196         .d_close =      xptclose,
 197         .d_ioctl =      xptioctl,
 198         .d_name =       "xpt",
 199 };
 200
 201 /* Storage for debugging datastructures */
 202 struct cam_path *cam_dpath;
 203 u_int32_t cam_dflags = CAM_DEBUG_FLAGS;
 204 SYSCTL_UINT(_kern_cam, OID_AUTO, dflags, CTLFLAG_RWTUN,
 205         &cam_dflags, 0, "Enabled debug flags");
 206 u_int32_t cam_debug_delay = CAM_DEBUG_DELAY;
 207 SYSCTL_UINT(_kern_cam, OID_AUTO, debug_delay, CTLFLAG_RWTUN,
 208         &cam_debug_delay, 0, "Delay in us after each debug message");
 209
 210 /* Our boot-time initialization hook */
 211 static int cam_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 212
 213 static moduledata_t cam_moduledata = {
 214         "cam",
 215         cam_module_event_handler,
 216         NULL
 217 };
 218
 219 static int      xpt_init(void *);
 220
 221 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
 222 MODULE_VERSION(cam, 1);
 223
 224
 225 static void             xpt_async_bcast(struct async_list *async_head,
 226                                         u_int32_t async_code,
 227                                         struct cam_path *path,
 228                                         void *async_arg);
 229 static path_id_t xptnextfreepathid(void);
 230 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
 231 static union ccb *xpt_get_ccb(struct cam_periph *periph);
 232 static union ccb *xpt_get_ccb_nowait(struct cam_periph *periph);
 233 static void      xpt_run_allocq(struct cam_periph *periph, int sleep);
 234 static void      xpt_run_allocq_task(void *context, int pending);
 235 static void      xpt_run_devq(struct cam_devq *devq);
 236 static timeout_t xpt_release_devq_timeout;
 237 static void      xpt_release_simq_timeout(void *arg) __unused;
 238 static void      xpt_acquire_bus(struct cam_eb *bus);
 239 static void      xpt_release_bus(struct cam_eb *bus);
 240 static uint32_t  xpt_freeze_devq_device(struct cam_ed *dev, u_int count);
 241 static int       xpt_release_devq_device(struct cam_ed *dev, u_int count,
 242                     int run_queue);
 243 static struct cam_et*
 244                  xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
 245 static void      xpt_acquire_target(struct cam_et *target);
 246 static void      xpt_release_target(struct cam_et *target);
 247 static struct cam_eb*
 248                  xpt_find_bus(path_id_t path_id);
 249 static struct cam_et*
 250                  xpt_find_target(struct cam_eb *bus, target_id_t target_id);
 251 static struct cam_ed*
 252                  xpt_find_device(struct cam_et *target, lun_id_t lun_id);
 253 static void      xpt_config(void *arg);
 254 static int       xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 255                                  u_int32_t new_priority);
 256 static xpt_devicefunc_t xptpassannouncefunc;
 257 static void      xptaction(struct cam_sim *sim, union ccb *work_ccb);
 258 static void      xptpoll(struct cam_sim *sim);
 259 static void      camisr_runqueue(void);
 260 static void      xpt_done_process(struct ccb_hdr *ccb_h);
 261 static void      xpt_done_td(void *);
 262 static dev_match_ret    xptbusmatch(struct dev_match_pattern *patterns,
 263                                     u_int num_patterns, struct cam_eb *bus);
 264 static dev_match_ret    xptdevicematch(struct dev_match_pattern *patterns,
 265                                        u_int num_patterns,
 266                                        struct cam_ed *device);
 267 static dev_match_ret    xptperiphmatch(struct dev_match_pattern *patterns,
 268                                        u_int num_patterns,
 269                                        struct cam_periph *periph);
 270 static xpt_busfunc_t    xptedtbusfunc;
 271 static xpt_targetfunc_t xptedttargetfunc;
 272 static xpt_devicefunc_t xptedtdevicefunc;
 273 static xpt_periphfunc_t xptedtperiphfunc;
 274 static xpt_pdrvfunc_t   xptplistpdrvfunc;
 275 static xpt_periphfunc_t xptplistperiphfunc;
 276 static int              xptedtmatch(struct ccb_dev_match *cdm);
 277 static int              xptperiphlistmatch(struct ccb_dev_match *cdm);
 278 static int              xptbustraverse(struct cam_eb *start_bus,
 279                                        xpt_busfunc_t *tr_func, void *arg);
 280 static int              xpttargettraverse(struct cam_eb *bus,
 281                                           struct cam_et *start_target,
 282                                           xpt_targetfunc_t *tr_func, void *arg);
 283 static int              xptdevicetraverse(struct cam_et *target,
 284                                           struct cam_ed *start_device,
 285                                           xpt_devicefunc_t *tr_func, void *arg);
 286 static int              xptperiphtraverse(struct cam_ed *device,
 287                                           struct cam_periph *start_periph,
 288                                           xpt_periphfunc_t *tr_func, void *arg);
 289 static int              xptpdrvtraverse(struct periph_driver **start_pdrv,
 290                                         xpt_pdrvfunc_t *tr_func, void *arg);
 291 static int              xptpdperiphtraverse(struct periph_driver **pdrv,
 292                                             struct cam_periph *start_periph,
 293                                             xpt_periphfunc_t *tr_func,
 294                                             void *arg);
 295 static xpt_busfunc_t    xptdefbusfunc;
 296 static xpt_targetfunc_t xptdeftargetfunc;
 297 static xpt_devicefunc_t xptdefdevicefunc;
 298 static xpt_periphfunc_t xptdefperiphfunc;
 299 static void             xpt_finishconfig_task(void *context, int pending);
 300 static void             xpt_dev_async_default(u_int32_t async_code,
 301                                               struct cam_eb *bus,
 302                                               struct cam_et *target,
 303                                               struct cam_ed *device,
 304                                               void *async_arg);
 305 static struct cam_ed *  xpt_alloc_device_default(struct cam_eb *bus,
 306                                                  struct cam_et *target,
 307                                                  lun_id_t lun_id);
 308 static xpt_devicefunc_t xptsetasyncfunc;
 309 static xpt_busfunc_t    xptsetasyncbusfunc;
 310 static cam_status       xptregister(struct cam_periph *periph,
 311                                     void *arg);
 312 static const char *     xpt_action_name(uint32_t action);
 313 static __inline int device_is_queued(struct cam_ed *device);
 314
 315 static __inline int
 316 xpt_schedule_devq(struct cam_devq *devq, struct cam_ed *dev)
 317 {
 318         int     retval;
 319
 320         mtx_assert(&devq->send_mtx, MA_OWNED);
 321         if ((dev->ccbq.queue.entries > 0) &&
 322             (dev->ccbq.dev_openings > 0) &&
 323             (dev->ccbq.queue.qfrozen_cnt == 0)) {
 324                 /*
 325                  * The priority of a device waiting for controller
 326                  * resources is that of the highest priority CCB
 327                  * enqueued.
 328                  */
 329                 retval =
 330                     xpt_schedule_dev(&devq->send_queue,
 331                                      &dev->devq_entry,
 332                                      CAMQ_GET_PRIO(&dev->ccbq.queue));
 333         } else {
 334                 retval = 0;
 335         }
 336         return (retval);
 337 }
 338
 339 static __inline int
 340 device_is_queued(struct cam_ed *device)
 341 {
 342         return (device->devq_entry.index != CAM_UNQUEUED_INDEX);
 343 }
 344
 345 static void
 346 xpt_periph_init()
 347 {
 348         make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
 349 }
 350
 351 static int
 352 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 353 {
 354
 355         /*
 356          * Only allow read-write access.
 357          */
 358         if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
 359                 return(EPERM);
 360
 361         /*
 362          * We don't allow nonblocking access.
 363          */
 364         if ((flags & O_NONBLOCK) != 0) {
 365                 printf("%s: can't do nonblocking access\n", devtoname(dev));
 366                 return(ENODEV);
 367         }
 368
 369         return(0);
 370 }
 371
 372 static int
 373 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 374 {
 375
 376         return(0);
 377 }
 378
 379 /*
 380  * Don't automatically grab the xpt softc lock here even though this is going
 381  * through the xpt device.  The xpt device is really just a back door for
 382  * accessing other devices and SIMs, so the right thing to do is to grab
 383  * the appropriate SIM lock once the bus/SIM is located.
 384  */
 385 static int
 386 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 387 {
 388         int error;
 389
 390         if ((error = xptdoioctl(dev, cmd, addr, flag, td)) == ENOTTY) {
 391                 error = cam_compat_ioctl(dev, cmd, addr, flag, td, xptdoioctl);
 392         }
 393         return (error);
 394 }
 395
 396 static int
 397 xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 398 {
 399         int error;
 400
 401         error = 0;
 402
 403         switch(cmd) {
 404         /*
 405          * For the transport layer CAMIOCOMMAND ioctl, we really only want
 406          * to accept CCB types that don't quite make sense to send through a
 407          * passthrough driver. XPT_PATH_INQ is an exception to this, as stated
 408          * in the CAM spec.
 409          */
 410         case CAMIOCOMMAND: {
 411                 union ccb *ccb;
 412                 union ccb *inccb;
 413                 struct cam_eb *bus;
 414
 415                 inccb = (union ccb *)addr;
 416
 417                 bus = xpt_find_bus(inccb->ccb_h.path_id);
 418                 if (bus == NULL)
 419                         return (EINVAL);
 420
 421                 switch (inccb->ccb_h.func_code) {
 422                 case XPT_SCAN_BUS:
 423                 case XPT_RESET_BUS:
 424                         if (inccb->ccb_h.target_id != CAM_TARGET_WILDCARD ||
 425                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 426                                 xpt_release_bus(bus);
 427                                 return (EINVAL);
 428                         }
 429                         break;
 430                 case XPT_SCAN_TGT:
 431                         if (inccb->ccb_h.target_id == CAM_TARGET_WILDCARD ||
 432                             inccb->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 433                                 xpt_release_bus(bus);
 434                                 return (EINVAL);
 435                         }
 436                         break;
 437                 default:
 438                         break;
 439                 }
 440
 441                 switch(inccb->ccb_h.func_code) {
 442                 case XPT_SCAN_BUS:
 443                 case XPT_RESET_BUS:
 444                 case XPT_PATH_INQ:
 445                 case XPT_ENG_INQ:
 446                 case XPT_SCAN_LUN:
 447                 case XPT_SCAN_TGT:
 448
 449                         ccb = xpt_alloc_ccb();
 450
 451                         /*
 452                          * Create a path using the bus, target, and lun the
 453                          * user passed in.
 454                          */
 455                         if (xpt_create_path(&ccb->ccb_h.path, NULL,
 456                                             inccb->ccb_h.path_id,
 457                                             inccb->ccb_h.target_id,
 458                                             inccb->ccb_h.target_lun) !=
 459                                             CAM_REQ_CMP){
 460                                 error = EINVAL;
 461                                 xpt_free_ccb(ccb);
 462                                 break;
 463                         }
 464                         /* Ensure all of our fields are correct */
 465                         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
 466                                       inccb->ccb_h.pinfo.priority);
 467                         xpt_merge_ccb(ccb, inccb);
 468                         xpt_path_lock(ccb->ccb_h.path);
 469                         cam_periph_runccb(ccb, NULL, 0, 0, NULL);
 470                         xpt_path_unlock(ccb->ccb_h.path);
 471                         bcopy(ccb, inccb, sizeof(union ccb));
 472                         xpt_free_path(ccb->ccb_h.path);
 473                         xpt_free_ccb(ccb);
 474                         break;
 475
 476                 case XPT_DEBUG: {
 477                         union ccb ccb;
 478
 479                         /*
 480                          * This is an immediate CCB, so it's okay to
 481                          * allocate it on the stack.
 482                          */
 483
 484                         /*
 485                          * Create a path using the bus, target, and lun the
 486                          * user passed in.
 487                          */
 488                         if (xpt_create_path(&ccb.ccb_h.path, NULL,
 489                                             inccb->ccb_h.path_id,
 490                                             inccb->ccb_h.target_id,
 491                                             inccb->ccb_h.target_lun) !=
 492                                             CAM_REQ_CMP){
 493                                 error = EINVAL;
 494                                 break;
 495                         }
 496                         /* Ensure all of our fields are correct */
 497                         xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
 498                                       inccb->ccb_h.pinfo.priority);
 499                         xpt_merge_ccb(&ccb, inccb);
 500                         xpt_action(&ccb);
 501                         bcopy(&ccb, inccb, sizeof(union ccb));
 502                         xpt_free_path(ccb.ccb_h.path);
 503                         break;
 504
 505                 }
 506                 case XPT_DEV_MATCH: {
 507                         struct cam_periph_map_info mapinfo;
 508                         struct cam_path *old_path;
 509
 510                         /*
 511                          * We can't deal with physical addresses for this
 512                          * type of transaction.
 513                          */
 514                         if ((inccb->ccb_h.flags & CAM_DATA_MASK) !=
 515                             CAM_DATA_VADDR) {
 516                                 error = EINVAL;
 517                                 break;
 518                         }
 519
 520                         /*
 521                          * Save this in case the caller had it set to
 522                          * something in particular.
 523                          */
 524                         old_path = inccb->ccb_h.path;
 525
 526                         /*
 527                          * We really don't need a path for the matching
 528                          * code.  The path is needed because of the
 529                          * debugging statements in xpt_action().  They
 530                          * assume that the CCB has a valid path.
 531                          */
 532                         inccb->ccb_h.path = xpt_periph->path;
 533
 534                         bzero(&mapinfo, sizeof(mapinfo));
 535
 536                         /*
 537                          * Map the pattern and match buffers into kernel
 538                          * virtual address space.
 539                          */
 540                         error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
 541
 542                         if (error) {
 543                                 inccb->ccb_h.path = old_path;
 544                                 break;
 545                         }
 546
 547                         /*
 548                          * This is an immediate CCB, we can send it on directly.
 549                          */
 550                         xpt_action(inccb);
 551
 552                         /*
 553                          * Map the buffers back into user space.
 554                          */
 555                         cam_periph_unmapmem(inccb, &mapinfo);
 556
 557                         inccb->ccb_h.path = old_path;
 558
 559                         error = 0;
 560                         break;
 561                 }
 562                 default:
 563                         error = ENOTSUP;
 564                         break;
 565                 }
 566                 xpt_release_bus(bus);
 567                 break;
 568         }
 569         /*
 570          * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
 571          * with the periphal driver name and unit name filled in.  The other
 572          * fields don't really matter as input.  The passthrough driver name
 573          * ("pass"), and unit number are passed back in the ccb.  The current
 574          * device generation number, and the index into the device peripheral
 575          * driver list, and the status are also passed back.  Note that
 576          * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
 577          * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
 578          * (or rather should be) impossible for the device peripheral driver
 579          * list to change since we look at the whole thing in one pass, and
 580          * we do it with lock protection.
 581          *
 582          */
 583         case CAMGETPASSTHRU: {
 584                 union ccb *ccb;
 585                 struct cam_periph *periph;
 586                 struct periph_driver **p_drv;
 587                 char   *name;
 588                 u_int unit;
 589                 int base_periph_found;
 590
 591                 ccb = (union ccb *)addr;
 592                 unit = ccb->cgdl.unit_number;
 593                 name = ccb->cgdl.periph_name;
 594                 base_periph_found = 0;
 595
 596                 /*
 597                  * Sanity check -- make sure we don't get a null peripheral
 598                  * driver name.
 599                  */
 600                 if (*ccb->cgdl.periph_name == '\0') {
 601                         error = EINVAL;
 602                         break;
 603                 }
 604
 605                 /* Keep the list from changing while we traverse it */
 606                 xpt_lock_buses();
 607
 608                 /* first find our driver in the list of drivers */
 609                 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
 610                         if (strcmp((*p_drv)->driver_name, name) == 0)
 611                                 break;
 612
 613                 if (*p_drv == NULL) {
 614                         xpt_unlock_buses();
 615                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 616                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 617                         *ccb->cgdl.periph_name = '\0';
 618                         ccb->cgdl.unit_number = 0;
 619                         error = ENOENT;
 620                         break;
 621                 }
 622
 623                 /*
 624                  * Run through every peripheral instance of this driver
 625                  * and check to see whether it matches the unit passed
 626                  * in by the user.  If it does, get out of the loops and
 627                  * find the passthrough driver associated with that
 628                  * peripheral driver.
 629                  */
 630                 for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
 631                      periph = TAILQ_NEXT(periph, unit_links)) {
 632
 633                         if (periph->unit_number == unit)
 634                                 break;
 635                 }
 636                 /*
 637                  * If we found the peripheral driver that the user passed
 638                  * in, go through all of the peripheral drivers for that
 639                  * particular device and look for a passthrough driver.
 640                  */
 641                 if (periph != NULL) {
 642                         struct cam_ed *device;
 643                         int i;
 644
 645                         base_periph_found = 1;
 646                         device = periph->path->device;
 647                         for (i = 0, periph = SLIST_FIRST(&device->periphs);
 648                              periph != NULL;
 649                              periph = SLIST_NEXT(periph, periph_links), i++) {
 650                                 /*
 651                                  * Check to see whether we have a
 652                                  * passthrough device or not.
 653                                  */
 654                                 if (strcmp(periph->periph_name, "pass") == 0) {
 655                                         /*
 656                                          * Fill in the getdevlist fields.
 657                                          */
 658                                         strlcpy(ccb->cgdl.periph_name,
 659                                                periph->periph_name,
 660                                                sizeof(ccb->cgdl.periph_name));
 661                                         ccb->cgdl.unit_number =
 662                                                 periph->unit_number;
 663                                         if (SLIST_NEXT(periph, periph_links))
 664                                                 ccb->cgdl.status =
 665                                                         CAM_GDEVLIST_MORE_DEVS;
 666                                         else
 667                                                 ccb->cgdl.status =
 668                                                        CAM_GDEVLIST_LAST_DEVICE;
 669                                         ccb->cgdl.generation =
 670                                                 device->generation;
 671                                         ccb->cgdl.index = i;
 672                                         /*
 673                                          * Fill in some CCB header fields
 674                                          * that the user may want.
 675                                          */
 676                                         ccb->ccb_h.path_id =
 677                                                 periph->path->bus->path_id;
 678                                         ccb->ccb_h.target_id =
 679                                                 periph->path->target->target_id;
 680                                         ccb->ccb_h.target_lun =
 681                                                 periph->path->device->lun_id;
 682                                         ccb->ccb_h.status = CAM_REQ_CMP;
 683                                         break;
 684                                 }
 685                         }
 686                 }
 687
 688                 /*
 689                  * If the periph is null here, one of two things has
 690                  * happened.  The first possibility is that we couldn't
 691                  * find the unit number of the particular peripheral driver
 692                  * that the user is asking about.  e.g. the user asks for
 693                  * the passthrough driver for "da11".  We find the list of
 694                  * "da" peripherals all right, but there is no unit 11.
 695                  * The other possibility is that we went through the list
 696                  * of peripheral drivers attached to the device structure,
 697                  * but didn't find one with the name "pass".  Either way,
 698                  * we return ENOENT, since we couldn't find something.
 699                  */
 700                 if (periph == NULL) {
 701                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 702                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
 703                         *ccb->cgdl.periph_name = '\0';
 704                         ccb->cgdl.unit_number = 0;
 705                         error = ENOENT;
 706                         /*
 707                          * It is unfortunate that this is even necessary,
 708                          * but there are many, many clueless users out there.
 709                          * If this is true, the user is looking for the
 710                          * passthrough driver, but doesn't have one in his
 711                          * kernel.
 712                          */
 713                         if (base_periph_found == 1) {
 714                                 printf("xptioctl: pass driver is not in the "
 715                                        "kernel\n");
 716                                 printf("xptioctl: put \"device pass\" in "
 717                                        "your kernel config file\n");
 718                         }
 719                 }
 720                 xpt_unlock_buses();
 721                 break;
 722                 }
 723         default:
 724                 error = ENOTTY;
 725                 break;
 726         }
 727
 728         return(error);
 729 }
 730
 731 static int
 732 cam_module_event_handler(module_t mod, int what, void *arg)
 733 {
 734         int error;
 735
 736         switch (what) {
 737         case MOD_LOAD:
 738                 if ((error = xpt_init(NULL)) != 0)
 739                         return (error);
 740                 break;
 741         case MOD_UNLOAD:
 742                 return EBUSY;
 743         default:
 744                 return EOPNOTSUPP;
 745         }
 746
 747         return 0;
 748 }
 749
 750 static void
 751 xpt_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
 752 {
 753
 754         if (done_ccb->ccb_h.ppriv_ptr1 == NULL) {
 755                 xpt_free_path(done_ccb->ccb_h.path);
 756                 xpt_free_ccb(done_ccb);
 757         } else {
 758                 done_ccb->ccb_h.cbfcnp = done_ccb->ccb_h.ppriv_ptr1;
 759                 (*done_ccb->ccb_h.cbfcnp)(periph, done_ccb);
 760         }
 761         xpt_release_boot();
 762 }
 763
 764 /* thread to handle bus rescans */
 765 static void
 766 xpt_scanner_thread(void *dummy)
 767 {
 768         union ccb       *ccb;
 769         struct cam_path  path;
 770
 771         xpt_lock_buses();
 772         for (;;) {
 773                 if (TAILQ_EMPTY(&xsoftc.ccb_scanq))
 774                         msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
 775                                "-", 0);
 776                 if ((ccb = (union ccb *)TAILQ_FIRST(&xsoftc.ccb_scanq)) != NULL) {
 777                         TAILQ_REMOVE(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 778                         xpt_unlock_buses();
 779
 780                         /*
 781                          * Since lock can be dropped inside and path freed
 782                          * by completion callback even before return here,
 783                          * take our own path copy for reference.
 784                          */
 785                         xpt_copy_path(&path, ccb->ccb_h.path);
 786                         xpt_path_lock(&path);
 787                         xpt_action(ccb);
 788                         xpt_path_unlock(&path);
 789                         xpt_release_path(&path);
 790
 791                         xpt_lock_buses();
 792                 }
 793         }
 794 }
 795
 796 void
 797 xpt_rescan(union ccb *ccb)
 798 {
 799         struct ccb_hdr *hdr;
 800
 801         /* Prepare request */
 802         if (ccb->ccb_h.path->target->target_id == CAM_TARGET_WILDCARD &&
 803             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 804                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 805         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 806             ccb->ccb_h.path->device->lun_id == CAM_LUN_WILDCARD)
 807                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 808         else if (ccb->ccb_h.path->target->target_id != CAM_TARGET_WILDCARD &&
 809             ccb->ccb_h.path->device->lun_id != CAM_LUN_WILDCARD)
 810                 ccb->ccb_h.func_code = XPT_SCAN_LUN;
 811         else {
 812                 xpt_print(ccb->ccb_h.path, "illegal scan path\n");
 813                 xpt_free_path(ccb->ccb_h.path);
 814                 xpt_free_ccb(ccb);
 815                 return;
 816         }
 817         CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
 818             ("xpt_rescan: func %#x %s\n", ccb->ccb_h.func_code,
 819                 xpt_action_name(ccb->ccb_h.func_code)));
 820
 821         ccb->ccb_h.ppriv_ptr1 = ccb->ccb_h.cbfcnp;
 822         ccb->ccb_h.cbfcnp = xpt_rescan_done;
 823         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, CAM_PRIORITY_XPT);
 824         /* Don't make duplicate entries for the same paths. */
 825         xpt_lock_buses();
 826         if (ccb->ccb_h.ppriv_ptr1 == NULL) {
 827                 TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
 828                         if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
 829                                 wakeup(&xsoftc.ccb_scanq);
 830                                 xpt_unlock_buses();
 831                                 xpt_print(ccb->ccb_h.path, "rescan already queued\n");
 832                                 xpt_free_path(ccb->ccb_h.path);
 833                                 xpt_free_ccb(ccb);
 834                                 return;
 835                         }
 836                 }
 837         }
 838         TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
 839         xsoftc.buses_to_config++;
 840         wakeup(&xsoftc.ccb_scanq);
 841         xpt_unlock_buses();
 842 }
 843
 844 /* Functions accessed by the peripheral drivers */
 845 static int
 846 xpt_init(void *dummy)
 847 {
 848         struct cam_sim *xpt_sim;
 849         struct cam_path *path;
 850         struct cam_devq *devq;
 851         cam_status status;
 852         int error, i;
 853
 854         TAILQ_INIT(&xsoftc.xpt_busses);
 855         TAILQ_INIT(&xsoftc.ccb_scanq);
 856         STAILQ_INIT(&xsoftc.highpowerq);
 857         xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
 858
 859         mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
 860         mtx_init(&xsoftc.xpt_highpower_lock, "XPT highpower lock", NULL, MTX_DEF);
 861         xsoftc.xpt_taskq = taskqueue_create("CAM XPT task", M_WAITOK,
 862             taskqueue_thread_enqueue, /*context*/&xsoftc.xpt_taskq);
 863
 864 #ifdef CAM_BOOT_DELAY
 865         /*
 866          * Override this value at compile time to assist our users
 867          * who don't use loader to boot a kernel.
 868          */
 869         xsoftc.boot_delay = CAM_BOOT_DELAY;
 870 #endif
 871         /*
 872          * The xpt layer is, itself, the equivalent of a SIM.
 873          * Allow 16 ccbs in the ccb pool for it.  This should
 874          * give decent parallelism when we probe busses and
 875          * perform other XPT functions.
 876          */
 877         devq = cam_simq_alloc(16);
 878         xpt_sim = cam_sim_alloc(xptaction,
 879                                 xptpoll,
 880                                 "xpt",
 881                                 /*softc*/NULL,
 882                                 /*unit*/0,
 883                                 /*mtx*/&xsoftc.xpt_lock,
 884                                 /*max_dev_transactions*/0,
 885                                 /*max_tagged_dev_transactions*/0,
 886                                 devq);
 887         if (xpt_sim == NULL)
 888                 return (ENOMEM);
 889
 890         mtx_lock(&xsoftc.xpt_lock);
 891         if ((status = xpt_bus_register(xpt_sim, NULL, 0)) != CAM_SUCCESS) {
 892                 mtx_unlock(&xsoftc.xpt_lock);
 893                 printf("xpt_init: xpt_bus_register failed with status %#x,"
 894                        " failing attach\n", status);
 895                 return (EINVAL);
 896         }
 897         mtx_unlock(&xsoftc.xpt_lock);
 898
 899         /*
 900          * Looking at the XPT from the SIM layer, the XPT is
 901          * the equivalent of a peripheral driver.  Allocate
 902          * a peripheral driver entry for us.
 903          */
 904         if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
 905                                       CAM_TARGET_WILDCARD,
 906                                       CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
 907                 printf("xpt_init: xpt_create_path failed with status %#x,"
 908                        " failing attach\n", status);
 909                 return (EINVAL);
 910         }
 911         xpt_path_lock(path);
 912         cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
 913                          path, NULL, 0, xpt_sim);
 914         xpt_path_unlock(path);
 915         xpt_free_path(path);
 916
 917         if (cam_num_doneqs < 1)
 918                 cam_num_doneqs = 1 + mp_ncpus / 6;
 919         else if (cam_num_doneqs > MAXCPU)
 920                 cam_num_doneqs = MAXCPU;
 921         for (i = 0; i < cam_num_doneqs; i++) {
 922                 mtx_init(&cam_doneqs[i].cam_doneq_mtx, "CAM doneq", NULL,
 923                     MTX_DEF);
 924                 STAILQ_INIT(&cam_doneqs[i].cam_doneq);
 925                 error = kproc_kthread_add(xpt_done_td, &cam_doneqs[i],
 926                     &cam_proc, NULL, 0, 0, "cam", "doneq%d", i);
 927                 if (error != 0) {
 928                         cam_num_doneqs = i;
 929                         break;
 930                 }
 931         }
 932         if (cam_num_doneqs < 1) {
 933                 printf("xpt_init: Cannot init completion queues "
 934                        "- failing attach\n");
 935                 return (ENOMEM);
 936         }
 937         /*
 938          * Register a callback for when interrupts are enabled.
 939          */
 940         xsoftc.xpt_config_hook =
 941             (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
 942                                               M_CAMXPT, M_NOWAIT | M_ZERO);
 943         if (xsoftc.xpt_config_hook == NULL) {
 944                 printf("xpt_init: Cannot malloc config hook "
 945                        "- failing attach\n");
 946                 return (ENOMEM);
 947         }
 948         xsoftc.xpt_config_hook->ich_func = xpt_config;
 949         if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
 950                 free (xsoftc.xpt_config_hook, M_CAMXPT);
 951                 printf("xpt_init: config_intrhook_establish failed "
 952                        "- failing attach\n");
 953         }
 954
 955         return (0);
 956 }
 957
 958 static cam_status
 959 xptregister(struct cam_periph *periph, void *arg)
 960 {
 961         struct cam_sim *xpt_sim;
 962
 963         if (periph == NULL) {
 964                 printf("xptregister: periph was NULL!!\n");
 965                 return(CAM_REQ_CMP_ERR);
 966         }
 967
 968         xpt_sim = (struct cam_sim *)arg;
 969         xpt_sim->softc = periph;
 970         xpt_periph = periph;
 971         periph->softc = NULL;
 972
 973         return(CAM_REQ_CMP);
 974 }
 975
 976 int32_t
 977 xpt_add_periph(struct cam_periph *periph)
 978 {
 979         struct cam_ed *device;
 980         int32_t  status;
 981
 982         TASK_INIT(&periph->periph_run_task, 0, xpt_run_allocq_task, periph);
 983         device = periph->path->device;
 984         status = CAM_REQ_CMP;
 985         if (device != NULL) {
 986                 mtx_lock(&device->target->bus->eb_mtx);
 987                 device->generation++;
 988                 SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
 989                 mtx_unlock(&device->target->bus->eb_mtx);
 990                 atomic_add_32(&xsoftc.xpt_generation, 1);
 991         }
 992
 993         return (status);
 994 }
 995
 996 void
 997 xpt_remove_periph(struct cam_periph *periph)
 998 {
 999         struct cam_ed *device;
1000
1001         device = periph->path->device;
1002         if (device != NULL) {
1003                 mtx_lock(&device->target->bus->eb_mtx);
1004                 device->generation++;
1005                 SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
1006                 mtx_unlock(&device->target->bus->eb_mtx);
1007                 atomic_add_32(&xsoftc.xpt_generation, 1);
1008         }
1009 }
1010
1011
1012 void
1013 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1014 {
1015         struct  cam_path *path = periph->path;
1016
1017         cam_periph_assert(periph, MA_OWNED);
1018         periph->flags |= CAM_PERIPH_ANNOUNCED;
1019
1020         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1021                periph->periph_name, periph->unit_number,
1022                path->bus->sim->sim_name,
1023                path->bus->sim->unit_number,
1024                path->bus->sim->bus_id,
1025                path->bus->path_id,
1026                path->target->target_id,
1027                (uintmax_t)path->device->lun_id);
1028         printf("%s%d: ", periph->periph_name, periph->unit_number);
1029         if (path->device->protocol == PROTO_SCSI)
1030                 scsi_print_inquiry(&path->device->inq_data);
1031         else if (path->device->protocol == PROTO_ATA ||
1032             path->device->protocol == PROTO_SATAPM)
1033                 ata_print_ident(&path->device->ident_data);
1034         else if (path->device->protocol == PROTO_SEMB)
1035                 semb_print_ident(
1036                     (struct sep_identify_data *)&path->device->ident_data);
1037         else if (path->device->protocol == PROTO_NVME)
1038                 nvme_print_ident(path->device->nvme_cdata, path->device->nvme_data);
1039         else
1040                 printf("Unknown protocol device\n");
1041         if (path->device->serial_num_len > 0) {
1042                 /* Don't wrap the screen  - print only the first 60 chars */
1043                 printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1044                        periph->unit_number, path->device->serial_num);
1045         }
1046         /* Announce transport details. */
1047         (*(path->bus->xport->announce))(periph);
1048         /* Announce command queueing. */
1049         if (path->device->inq_flags & SID_CmdQue
1050          || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1051                 printf("%s%d: Command Queueing enabled\n",
1052                        periph->periph_name, periph->unit_number);
1053         }
1054         /* Announce caller's details if they've passed in. */
1055         if (announce_string != NULL)
1056                 printf("%s%d: %s\n", periph->periph_name,
1057                        periph->unit_number, announce_string);
1058 }
1059
1060 void
1061 xpt_announce_quirks(struct cam_periph *periph, int quirks, char *bit_string)
1062 {
1063         if (quirks != 0) {
1064                 printf("%s%d: quirks=0x%b\n", periph->periph_name,
1065                     periph->unit_number, quirks, bit_string);
1066         }
1067 }
1068
1069 void
1070 xpt_denounce_periph(struct cam_periph *periph)
1071 {
1072         struct  cam_path *path = periph->path;
1073
1074         cam_periph_assert(periph, MA_OWNED);
1075         printf("%s%d at %s%d bus %d scbus%d target %d lun %jx\n",
1076                periph->periph_name, periph->unit_number,
1077                path->bus->sim->sim_name,
1078                path->bus->sim->unit_number,
1079                path->bus->sim->bus_id,
1080                path->bus->path_id,
1081                path->target->target_id,
1082                (uintmax_t)path->device->lun_id);
1083         printf("%s%d: ", periph->periph_name, periph->unit_number);
1084         if (path->device->protocol == PROTO_SCSI)
1085                 scsi_print_inquiry_short(&path->device->inq_data);
1086         else if (path->device->protocol == PROTO_ATA ||
1087             path->device->protocol == PROTO_SATAPM)
1088                 ata_print_ident_short(&path->device->ident_data);
1089         else if (path->device->protocol == PROTO_SEMB)
1090                 semb_print_ident_short(
1091                     (struct sep_identify_data *)&path->device->ident_data);
1092         else if (path->device->protocol == PROTO_NVME)
1093                 nvme_print_ident(path->device->nvme_cdata, path->device->nvme_data);
1094         else
1095                 printf("Unknown protocol device");
1096         if (path->device->serial_num_len > 0)
1097                 printf(" s/n %.60s", path->device->serial_num);
1098         printf(" detached\n");
1099 }
1100
1101
1102 int
1103 xpt_getattr(char *buf, size_t len, const char *attr, struct cam_path *path)
1104 {
1105         int ret = -1, l, o;
1106         struct ccb_dev_advinfo cdai;
1107         struct scsi_vpd_id_descriptor *idd;
1108
1109         xpt_path_assert(path, MA_OWNED);
1110
1111         memset(&cdai, 0, sizeof(cdai));
1112         xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
1113         cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
1114         cdai.flags = CDAI_FLAG_NONE;
1115         cdai.bufsiz = len;
1116
1117         if (!strcmp(attr, "GEOM::ident"))
1118                 cdai.buftype = CDAI_TYPE_SERIAL_NUM;
1119         else if (!strcmp(attr, "GEOM::physpath"))
1120                 cdai.buftype = CDAI_TYPE_PHYS_PATH;
1121         else if (strcmp(attr, "GEOM::lunid") == 0 ||
1122                  strcmp(attr, "GEOM::lunname") == 0) {
1123                 cdai.buftype = CDAI_TYPE_SCSI_DEVID;
1124                 cdai.bufsiz = CAM_SCSI_DEVID_MAXLEN;
1125         } else
1126                 goto out;
1127
1128         cdai.buf = malloc(cdai.bufsiz, M_CAMXPT, M_NOWAIT|M_ZERO);
1129         if (cdai.buf == NULL) {
1130                 ret = ENOMEM;
1131                 goto out;
1132         }
1133         xpt_action((union ccb *)&cdai); /* can only be synchronous */
1134         if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
1135                 cam_release_devq(cdai.ccb_h.path, 0, 0, 0, FALSE);
1136         if (cdai.provsiz == 0)
1137                 goto out;
1138         if (cdai.buftype == CDAI_TYPE_SCSI_DEVID) {
1139                 if (strcmp(attr, "GEOM::lunid") == 0) {
1140                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1141                             cdai.provsiz, scsi_devid_is_lun_naa);
1142                         if (idd == NULL)
1143                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1144                                     cdai.provsiz, scsi_devid_is_lun_eui64);
1145                         if (idd == NULL)
1146                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1147                                     cdai.provsiz, scsi_devid_is_lun_uuid);
1148                         if (idd == NULL)
1149                                 idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1150                                     cdai.provsiz, scsi_devid_is_lun_md5);
1151                 } else
1152                         idd = NULL;
1153                 if (idd == NULL)
1154                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1155                             cdai.provsiz, scsi_devid_is_lun_t10);
1156                 if (idd == NULL)
1157                         idd = scsi_get_devid((struct scsi_vpd_device_id *)cdai.buf,
1158                             cdai.provsiz, scsi_devid_is_lun_name);
1159                 if (idd == NULL)
1160                         goto out;
1161                 ret = 0;
1162                 if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_ASCII) {
1163                         if (idd->length < len) {
1164                                 for (l = 0; l < idd->length; l++)
1165                                         buf[l] = idd->identifier[l] ?
1166                                             idd->identifier[l] : ' ';
1167                                 buf[l] = 0;
1168                         } else
1169                                 ret = EFAULT;
1170                 } else if ((idd->proto_codeset & SVPD_ID_CODESET_MASK) == SVPD_ID_CODESET_UTF8) {
1171                         l = strnlen(idd->identifier, idd->length);
1172                         if (l < len) {
1173                                 bcopy(idd->identifier, buf, l);
1174                                 buf[l] = 0;
1175                         } else
1176                                 ret = EFAULT;
1177                 } else if ((idd->id_type & SVPD_ID_TYPE_MASK) == SVPD_ID_TYPE_UUID
1178                     && idd->identifier[0] == 0x10) {
1179                         if ((idd->length - 2) * 2 + 4 < len) {
1180                                 for (l = 2, o = 0; l < idd->length; l++) {
1181                                         if (l == 6 || l == 8 || l == 10 || l == 12)
1182                                             o += sprintf(buf + o, "-");
1183                                         o += sprintf(buf + o, "%02x",
1184                                             idd->identifier[l]);
1185                                 }
1186                         } else
1187                                 ret = EFAULT;
1188                 } else {
1189                         if (idd->length * 2 < len) {
1190                                 for (l = 0; l < idd->length; l++)
1191                                         sprintf(buf + l * 2, "%02x",
1192                                             idd->identifier[l]);
1193                         } else
1194                                 ret = EFAULT;
1195                 }
1196         } else {
1197                 ret = 0;
1198                 if (strlcpy(buf, cdai.buf, len) >= len)
1199                         ret = EFAULT;
1200         }
1201
1202 out:
1203         if (cdai.buf != NULL)
1204                 free(cdai.buf, M_CAMXPT);
1205         return ret;
1206 }
1207
1208 static dev_match_ret
1209 xptbusmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1210             struct cam_eb *bus)
1211 {
1212         dev_match_ret retval;
1213         u_int i;
1214
1215         retval = DM_RET_NONE;
1216
1217         /*
1218          * If we aren't given something to match against, that's an error.
1219          */
1220         if (bus == NULL)
1221                 return(DM_RET_ERROR);
1222
1223         /*
1224          * If there are no match entries, then this bus matches no
1225          * matter what.
1226          */
1227         if ((patterns == NULL) || (num_patterns == 0))
1228                 return(DM_RET_DESCEND | DM_RET_COPY);
1229
1230         for (i = 0; i < num_patterns; i++) {
1231                 struct bus_match_pattern *cur_pattern;
1232
1233                 /*
1234                  * If the pattern in question isn't for a bus node, we
1235                  * aren't interested.  However, we do indicate to the
1236                  * calling routine that we should continue descending the
1237                  * tree, since the user wants to match against lower-level
1238                  * EDT elements.
1239                  */
1240                 if (patterns[i].type != DEV_MATCH_BUS) {
1241                         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1242                                 retval |= DM_RET_DESCEND;
1243                         continue;
1244                 }
1245
1246                 cur_pattern = &patterns[i].pattern.bus_pattern;
1247
1248                 /*
1249                  * If they want to match any bus node, we give them any
1250                  * device node.
1251                  */
1252                 if (cur_pattern->flags == BUS_MATCH_ANY) {
1253                         /* set the copy flag */
1254                         retval |= DM_RET_COPY;
1255
1256                         /*
1257                          * If we've already decided on an action, go ahead
1258                          * and return.
1259                          */
1260                         if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1261                                 return(retval);
1262                 }
1263
1264                 /*
1265                  * Not sure why someone would do this...
1266                  */
1267                 if (cur_pattern->flags == BUS_MATCH_NONE)
1268                         continue;
1269
1270                 if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1271                  && (cur_pattern->path_id != bus->path_id))
1272                         continue;
1273
1274                 if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1275                  && (cur_pattern->bus_id != bus->sim->bus_id))
1276                         continue;
1277
1278                 if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1279                  && (cur_pattern->unit_number != bus->sim->unit_number))
1280                         continue;
1281
1282                 if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1283                  && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1284                              DEV_IDLEN) != 0))
1285                         continue;
1286
1287                 /*
1288                  * If we get to this point, the user definitely wants
1289                  * information on this bus.  So tell the caller to copy the
1290                  * data out.
1291                  */
1292                 retval |= DM_RET_COPY;
1293
1294                 /*
1295                  * If the return action has been set to descend, then we
1296                  * know that we've already seen a non-bus matching
1297                  * expression, therefore we need to further descend the tree.
1298                  * This won't change by continuing around the loop, so we
1299                  * go ahead and return.  If we haven't seen a non-bus
1300                  * matching expression, we keep going around the loop until
1301                  * we exhaust the matching expressions.  We'll set the stop
1302                  * flag once we fall out of the loop.
1303                  */
1304                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1305                         return(retval);
1306         }
1307
1308         /*
1309          * If the return action hasn't been set to descend yet, that means
1310          * we haven't seen anything other than bus matching patterns.  So
1311          * tell the caller to stop descending the tree -- the user doesn't
1312          * want to match against lower level tree elements.
1313          */
1314         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1315                 retval |= DM_RET_STOP;
1316
1317         return(retval);
1318 }
1319
1320 static dev_match_ret
1321 xptdevicematch(struct dev_match_pattern *patterns, u_int num_patterns,
1322                struct cam_ed *device)
1323 {
1324         dev_match_ret retval;
1325         u_int i;
1326
1327         retval = DM_RET_NONE;
1328
1329         /*
1330          * If we aren't given something to match against, that's an error.
1331          */
1332         if (device == NULL)
1333                 return(DM_RET_ERROR);
1334
1335         /*
1336          * If there are no match entries, then this device matches no
1337          * matter what.
1338          */
1339         if ((patterns == NULL) || (num_patterns == 0))
1340                 return(DM_RET_DESCEND | DM_RET_COPY);
1341
1342         for (i = 0; i < num_patterns; i++) {
1343                 struct device_match_pattern *cur_pattern;
1344                 struct scsi_vpd_device_id *device_id_page;
1345
1346                 /*
1347                  * If the pattern in question isn't for a device node, we
1348                  * aren't interested.
1349                  */
1350                 if (patterns[i].type != DEV_MATCH_DEVICE) {
1351                         if ((patterns[i].type == DEV_MATCH_PERIPH)
1352                          && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1353                                 retval |= DM_RET_DESCEND;
1354                         continue;
1355                 }
1356
1357                 cur_pattern = &patterns[i].pattern.device_pattern;
1358
1359                 /* Error out if mutually exclusive options are specified. */
1360                 if ((cur_pattern->flags & (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1361                  == (DEV_MATCH_INQUIRY|DEV_MATCH_DEVID))
1362                         return(DM_RET_ERROR);
1363
1364                 /*
1365                  * If they want to match any device node, we give them any
1366                  * device node.
1367                  */
1368                 if (cur_pattern->flags == DEV_MATCH_ANY)
1369                         goto copy_dev_node;
1370
1371                 /*
1372                  * Not sure why someone would do this...
1373                  */
1374                 if (cur_pattern->flags == DEV_MATCH_NONE)
1375                         continue;
1376
1377                 if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1378                  && (cur_pattern->path_id != device->target->bus->path_id))
1379                         continue;
1380
1381                 if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1382                  && (cur_pattern->target_id != device->target->target_id))
1383                         continue;
1384
1385                 if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1386                  && (cur_pattern->target_lun != device->lun_id))
1387                         continue;
1388
1389                 if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1390                  && (cam_quirkmatch((caddr_t)&device->inq_data,
1391                                     (caddr_t)&cur_pattern->data.inq_pat,
1392                                     1, sizeof(cur_pattern->data.inq_pat),
1393                                     scsi_static_inquiry_match) == NULL))
1394                         continue;
1395
1396                 device_id_page = (struct scsi_vpd_device_id *)device->device_id;
1397                 if (((cur_pattern->flags & DEV_MATCH_DEVID) != 0)
1398                  && (device->device_id_len < SVPD_DEVICE_ID_HDR_LEN
1399                   || scsi_devid_match((uint8_t *)device_id_page->desc_list,
1400                                       device->device_id_len
1401                                     - SVPD_DEVICE_ID_HDR_LEN,
1402                                       cur_pattern->data.devid_pat.id,
1403                                       cur_pattern->data.devid_pat.id_len) != 0))
1404                         continue;
1405
1406 copy_dev_node:
1407                 /*
1408                  * If we get to this point, the user definitely wants
1409                  * information on this device.  So tell the caller to copy
1410                  * the data out.
1411                  */
1412                 retval |= DM_RET_COPY;
1413
1414                 /*
1415                  * If the return action has been set to descend, then we
1416                  * know that we've already seen a peripheral matching
1417                  * expression, therefore we need to further descend the tree.
1418                  * This won't change by continuing around the loop, so we
1419                  * go ahead and return.  If we haven't seen a peripheral
1420                  * matching expression, we keep going around the loop until
1421                  * we exhaust the matching expressions.  We'll set the stop
1422                  * flag once we fall out of the loop.
1423                  */
1424                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1425                         return(retval);
1426         }
1427
1428         /*
1429          * If the return action hasn't been set to descend yet, that means
1430          * we haven't seen any peripheral matching patterns.  So tell the
1431          * caller to stop descending the tree -- the user doesn't want to
1432          * match against lower level tree elements.
1433          */
1434         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1435                 retval |= DM_RET_STOP;
1436
1437         return(retval);
1438 }
1439
1440 /*
1441  * Match a single peripheral against any number of match patterns.
1442  */
1443 static dev_match_ret
1444 xptperiphmatch(struct dev_match_pattern *patterns, u_int num_patterns,
1445                struct cam_periph *periph)
1446 {
1447         dev_match_ret retval;
1448         u_int i;
1449
1450         /*
1451          * If we aren't given something to match against, that's an error.
1452          */
1453         if (periph == NULL)
1454                 return(DM_RET_ERROR);
1455
1456         /*
1457          * If there are no match entries, then this peripheral matches no
1458          * matter what.
1459          */
1460         if ((patterns == NULL) || (num_patterns == 0))
1461                 return(DM_RET_STOP | DM_RET_COPY);
1462
1463         /*
1464          * There aren't any nodes below a peripheral node, so there's no
1465          * reason to descend the tree any further.
1466          */
1467         retval = DM_RET_STOP;
1468
1469         for (i = 0; i < num_patterns; i++) {
1470                 struct periph_match_pattern *cur_pattern;
1471
1472                 /*
1473                  * If the pattern in question isn't for a peripheral, we
1474                  * aren't interested.
1475                  */
1476                 if (patterns[i].type != DEV_MATCH_PERIPH)
1477                         continue;
1478
1479                 cur_pattern = &patterns[i].pattern.periph_pattern;
1480
1481                 /*
1482                  * If they want to match on anything, then we will do so.
1483                  */
1484                 if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1485                         /* set the copy flag */
1486                         retval |= DM_RET_COPY;
1487
1488                         /*
1489                          * We've already set the return action to stop,
1490                          * since there are no nodes below peripherals in
1491                          * the tree.
1492                          */
1493                         return(retval);
1494                 }
1495
1496                 /*
1497                  * Not sure why someone would do this...
1498                  */
1499                 if (cur_pattern->flags == PERIPH_MATCH_NONE)
1500                         continue;
1501
1502                 if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1503                  && (cur_pattern->path_id != periph->path->bus->path_id))
1504                         continue;
1505
1506                 /*
1507                  * For the target and lun id's, we have to make sure the
1508                  * target and lun pointers aren't NULL.  The xpt peripheral
1509                  * has a wildcard target and device.
1510                  */
1511                 if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1512                  && ((periph->path->target == NULL)
1513                  ||(cur_pattern->target_id != periph->path->target->target_id)))
1514                         continue;
1515
1516                 if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1517                  && ((periph->path->device == NULL)
1518                  || (cur_pattern->target_lun != periph->path->device->lun_id)))
1519                         continue;
1520
1521                 if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1522                  && (cur_pattern->unit_number != periph->unit_number))
1523                         continue;
1524
1525                 if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1526                  && (strncmp(cur_pattern->periph_name, periph->periph_name,
1527                              DEV_IDLEN) != 0))
1528                         continue;
1529
1530                 /*
1531                  * If we get to this point, the user definitely wants
1532                  * information on this peripheral.  So tell the caller to
1533                  * copy the data out.
1534                  */
1535                 retval |= DM_RET_COPY;
1536
1537                 /*
1538                  * The return action has already been set to stop, since
1539                  * peripherals don't have any nodes below them in the EDT.
1540                  */
1541                 return(retval);
1542         }
1543
1544         /*
1545          * If we get to this point, the peripheral that was passed in
1546          * doesn't match any of the patterns.
1547          */
1548         return(retval);
1549 }
1550
1551 static int
1552 xptedtbusfunc(struct cam_eb *bus, void *arg)
1553 {
1554         struct ccb_dev_match *cdm;
1555         struct cam_et *target;
1556         dev_match_ret retval;
1557
1558         cdm = (struct ccb_dev_match *)arg;
1559
1560         /*
1561          * If our position is for something deeper in the tree, that means
1562          * that we've already seen this node.  So, we keep going down.
1563          */
1564         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1565          && (cdm->pos.cookie.bus == bus)
1566          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1567          && (cdm->pos.cookie.target != NULL))
1568                 retval = DM_RET_DESCEND;
1569         else
1570                 retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1571
1572         /*
1573          * If we got an error, bail out of the search.
1574          */
1575         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1576                 cdm->status = CAM_DEV_MATCH_ERROR;
1577                 return(0);
1578         }
1579
1580         /*
1581          * If the copy flag is set, copy this bus out.
1582          */
1583         if (retval & DM_RET_COPY) {
1584                 int spaceleft, j;
1585
1586                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1587                         sizeof(struct dev_match_result));
1588
1589                 /*
1590                  * If we don't have enough space to put in another
1591                  * match result, save our position and tell the
1592                  * user there are more devices to check.
1593                  */
1594                 if (spaceleft < sizeof(struct dev_match_result)) {
1595                         bzero(&cdm->pos, sizeof(cdm->pos));
1596                         cdm->pos.position_type =
1597                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1598
1599                         cdm->pos.cookie.bus = bus;
1600                         cdm->pos.generations[CAM_BUS_GENERATION]=
1601                                 xsoftc.bus_generation;
1602                         cdm->status = CAM_DEV_MATCH_MORE;
1603                         return(0);
1604                 }
1605                 j = cdm->num_matches;
1606                 cdm->num_matches++;
1607                 cdm->matches[j].type = DEV_MATCH_BUS;
1608                 cdm->matches[j].result.bus_result.path_id = bus->path_id;
1609                 cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1610                 cdm->matches[j].result.bus_result.unit_number =
1611                         bus->sim->unit_number;
1612                 strlcpy(cdm->matches[j].result.bus_result.dev_name,
1613                         bus->sim->sim_name,
1614                         sizeof(cdm->matches[j].result.bus_result.dev_name));
1615         }
1616
1617         /*
1618          * If the user is only interested in busses, there's no
1619          * reason to descend to the next level in the tree.
1620          */
1621         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1622                 return(1);
1623
1624         /*
1625          * If there is a target generation recorded, check it to
1626          * make sure the target list hasn't changed.
1627          */
1628         mtx_lock(&bus->eb_mtx);
1629         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1630          && (cdm->pos.cookie.bus == bus)
1631          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1632          && (cdm->pos.cookie.target != NULL)) {
1633                 if ((cdm->pos.generations[CAM_TARGET_GENERATION] !=
1634                     bus->generation)) {
1635                         mtx_unlock(&bus->eb_mtx);
1636                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1637                         return (0);
1638                 }
1639                 target = (struct cam_et *)cdm->pos.cookie.target;
1640                 target->refcount++;
1641         } else
1642                 target = NULL;
1643         mtx_unlock(&bus->eb_mtx);
1644
1645         return (xpttargettraverse(bus, target, xptedttargetfunc, arg));
1646 }
1647
1648 static int
1649 xptedttargetfunc(struct cam_et *target, void *arg)
1650 {
1651         struct ccb_dev_match *cdm;
1652         struct cam_eb *bus;
1653         struct cam_ed *device;
1654
1655         cdm = (struct ccb_dev_match *)arg;
1656         bus = target->bus;
1657
1658         /*
1659          * If there is a device list generation recorded, check it to
1660          * make sure the device list hasn't changed.
1661          */
1662         mtx_lock(&bus->eb_mtx);
1663         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1664          && (cdm->pos.cookie.bus == bus)
1665          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1666          && (cdm->pos.cookie.target == target)
1667          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1668          && (cdm->pos.cookie.device != NULL)) {
1669                 if (cdm->pos.generations[CAM_DEV_GENERATION] !=
1670                     target->generation) {
1671                         mtx_unlock(&bus->eb_mtx);
1672                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1673                         return(0);
1674                 }
1675                 device = (struct cam_ed *)cdm->pos.cookie.device;
1676                 device->refcount++;
1677         } else
1678                 device = NULL;
1679         mtx_unlock(&bus->eb_mtx);
1680
1681         return (xptdevicetraverse(target, device, xptedtdevicefunc, arg));
1682 }
1683
1684 static int
1685 xptedtdevicefunc(struct cam_ed *device, void *arg)
1686 {
1687         struct cam_eb *bus;
1688         struct cam_periph *periph;
1689         struct ccb_dev_match *cdm;
1690         dev_match_ret retval;
1691
1692         cdm = (struct ccb_dev_match *)arg;
1693         bus = device->target->bus;
1694
1695         /*
1696          * If our position is for something deeper in the tree, that means
1697          * that we've already seen this node.  So, we keep going down.
1698          */
1699         if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1700          && (cdm->pos.cookie.device == device)
1701          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1702          && (cdm->pos.cookie.periph != NULL))
1703                 retval = DM_RET_DESCEND;
1704         else
1705                 retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
1706                                         device);
1707
1708         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1709                 cdm->status = CAM_DEV_MATCH_ERROR;
1710                 return(0);
1711         }
1712
1713         /*
1714          * If the copy flag is set, copy this device out.
1715          */
1716         if (retval & DM_RET_COPY) {
1717                 int spaceleft, j;
1718
1719                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1720                         sizeof(struct dev_match_result));
1721
1722                 /*
1723                  * If we don't have enough space to put in another
1724                  * match result, save our position and tell the
1725                  * user there are more devices to check.
1726                  */
1727                 if (spaceleft < sizeof(struct dev_match_result)) {
1728                         bzero(&cdm->pos, sizeof(cdm->pos));
1729                         cdm->pos.position_type =
1730                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1731                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
1732
1733                         cdm->pos.cookie.bus = device->target->bus;
1734                         cdm->pos.generations[CAM_BUS_GENERATION]=
1735                                 xsoftc.bus_generation;
1736                         cdm->pos.cookie.target = device->target;
1737                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1738                                 device->target->bus->generation;
1739                         cdm->pos.cookie.device = device;
1740                         cdm->pos.generations[CAM_DEV_GENERATION] =
1741                                 device->target->generation;
1742                         cdm->status = CAM_DEV_MATCH_MORE;
1743                         return(0);
1744                 }
1745                 j = cdm->num_matches;
1746                 cdm->num_matches++;
1747                 cdm->matches[j].type = DEV_MATCH_DEVICE;
1748                 cdm->matches[j].result.device_result.path_id =
1749                         device->target->bus->path_id;
1750                 cdm->matches[j].result.device_result.target_id =
1751                         device->target->target_id;
1752                 cdm->matches[j].result.device_result.target_lun =
1753                         device->lun_id;
1754                 cdm->matches[j].result.device_result.protocol =
1755                         device->protocol;
1756                 bcopy(&device->inq_data,
1757                       &cdm->matches[j].result.device_result.inq_data,
1758                       sizeof(struct scsi_inquiry_data));
1759                 bcopy(&device->ident_data,
1760                       &cdm->matches[j].result.device_result.ident_data,
1761                       sizeof(struct ata_params));
1762
1763                 /* Let the user know whether this device is unconfigured */
1764                 if (device->flags & CAM_DEV_UNCONFIGURED)
1765                         cdm->matches[j].result.device_result.flags =
1766                                 DEV_RESULT_UNCONFIGURED;
1767                 else
1768                         cdm->matches[j].result.device_result.flags =
1769                                 DEV_RESULT_NOFLAG;
1770         }
1771
1772         /*
1773          * If the user isn't interested in peripherals, don't descend
1774          * the tree any further.
1775          */
1776         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1777                 return(1);
1778
1779         /*
1780          * If there is a peripheral list generation recorded, make sure
1781          * it hasn't changed.
1782          */
1783         xpt_lock_buses();
1784         mtx_lock(&bus->eb_mtx);
1785         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1786          && (cdm->pos.cookie.bus == bus)
1787          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1788          && (cdm->pos.cookie.target == device->target)
1789          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1790          && (cdm->pos.cookie.device == device)
1791          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1792          && (cdm->pos.cookie.periph != NULL)) {
1793                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1794                     device->generation) {
1795                         mtx_unlock(&bus->eb_mtx);
1796                         xpt_unlock_buses();
1797                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1798                         return(0);
1799                 }
1800                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1801                 periph->refcount++;
1802         } else
1803                 periph = NULL;
1804         mtx_unlock(&bus->eb_mtx);
1805         xpt_unlock_buses();
1806
1807         return (xptperiphtraverse(device, periph, xptedtperiphfunc, arg));
1808 }
1809
1810 static int
1811 xptedtperiphfunc(struct cam_periph *periph, void *arg)
1812 {
1813         struct ccb_dev_match *cdm;
1814         dev_match_ret retval;
1815
1816         cdm = (struct ccb_dev_match *)arg;
1817
1818         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1819
1820         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1821                 cdm->status = CAM_DEV_MATCH_ERROR;
1822                 return(0);
1823         }
1824
1825         /*
1826          * If the copy flag is set, copy this peripheral out.
1827          */
1828         if (retval & DM_RET_COPY) {
1829                 int spaceleft, j;
1830                 size_t l;
1831
1832                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1833                         sizeof(struct dev_match_result));
1834
1835                 /*
1836                  * If we don't have enough space to put in another
1837                  * match result, save our position and tell the
1838                  * user there are more devices to check.
1839                  */
1840                 if (spaceleft < sizeof(struct dev_match_result)) {
1841                         bzero(&cdm->pos, sizeof(cdm->pos));
1842                         cdm->pos.position_type =
1843                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
1844                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
1845                                 CAM_DEV_POS_PERIPH;
1846
1847                         cdm->pos.cookie.bus = periph->path->bus;
1848                         cdm->pos.generations[CAM_BUS_GENERATION]=
1849                                 xsoftc.bus_generation;
1850                         cdm->pos.cookie.target = periph->path->target;
1851                         cdm->pos.generations[CAM_TARGET_GENERATION] =
1852                                 periph->path->bus->generation;
1853                         cdm->pos.cookie.device = periph->path->device;
1854                         cdm->pos.generations[CAM_DEV_GENERATION] =
1855                                 periph->path->target->generation;
1856                         cdm->pos.cookie.periph = periph;
1857                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
1858                                 periph->path->device->generation;
1859                         cdm->status = CAM_DEV_MATCH_MORE;
1860                         return(0);
1861                 }
1862
1863                 j = cdm->num_matches;
1864                 cdm->num_matches++;
1865                 cdm->matches[j].type = DEV_MATCH_PERIPH;
1866                 cdm->matches[j].result.periph_result.path_id =
1867                         periph->path->bus->path_id;
1868                 cdm->matches[j].result.periph_result.target_id =
1869                         periph->path->target->target_id;
1870                 cdm->matches[j].result.periph_result.target_lun =
1871                         periph->path->device->lun_id;
1872                 cdm->matches[j].result.periph_result.unit_number =
1873                         periph->unit_number;
1874                 l = sizeof(cdm->matches[j].result.periph_result.periph_name);
1875                 strlcpy(cdm->matches[j].result.periph_result.periph_name,
1876                         periph->periph_name, l);
1877         }
1878
1879         return(1);
1880 }
1881
1882 static int
1883 xptedtmatch(struct ccb_dev_match *cdm)
1884 {
1885         struct cam_eb *bus;
1886         int ret;
1887
1888         cdm->num_matches = 0;
1889
1890         /*
1891          * Check the bus list generation.  If it has changed, the user
1892          * needs to reset everything and start over.
1893          */
1894         xpt_lock_buses();
1895         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1896          && (cdm->pos.cookie.bus != NULL)) {
1897                 if (cdm->pos.generations[CAM_BUS_GENERATION] !=
1898                     xsoftc.bus_generation) {
1899                         xpt_unlock_buses();
1900                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1901                         return(0);
1902                 }
1903                 bus = (struct cam_eb *)cdm->pos.cookie.bus;
1904                 bus->refcount++;
1905         } else
1906                 bus = NULL;
1907         xpt_unlock_buses();
1908
1909         ret = xptbustraverse(bus, xptedtbusfunc, cdm);
1910
1911         /*
1912          * If we get back 0, that means that we had to stop before fully
1913          * traversing the EDT.  It also means that one of the subroutines
1914          * has set the status field to the proper value.  If we get back 1,
1915          * we've fully traversed the EDT and copied out any matching entries.
1916          */
1917         if (ret == 1)
1918                 cdm->status = CAM_DEV_MATCH_LAST;
1919
1920         return(ret);
1921 }
1922
1923 static int
1924 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
1925 {
1926         struct cam_periph *periph;
1927         struct ccb_dev_match *cdm;
1928
1929         cdm = (struct ccb_dev_match *)arg;
1930
1931         xpt_lock_buses();
1932         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
1933          && (cdm->pos.cookie.pdrv == pdrv)
1934          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
1935          && (cdm->pos.cookie.periph != NULL)) {
1936                 if (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
1937                     (*pdrv)->generation) {
1938                         xpt_unlock_buses();
1939                         cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1940                         return(0);
1941                 }
1942                 periph = (struct cam_periph *)cdm->pos.cookie.periph;
1943                 periph->refcount++;
1944         } else
1945                 periph = NULL;
1946         xpt_unlock_buses();
1947
1948         return (xptpdperiphtraverse(pdrv, periph, xptplistperiphfunc, arg));
1949 }
1950
1951 static int
1952 xptplistperiphfunc(struct cam_periph *periph, void *arg)
1953 {
1954         struct ccb_dev_match *cdm;
1955         dev_match_ret retval;
1956
1957         cdm = (struct ccb_dev_match *)arg;
1958
1959         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
1960
1961         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1962                 cdm->status = CAM_DEV_MATCH_ERROR;
1963                 return(0);
1964         }
1965
1966         /*
1967          * If the copy flag is set, copy this peripheral out.
1968          */
1969         if (retval & DM_RET_COPY) {
1970                 int spaceleft, j;
1971                 size_t l;
1972
1973                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1974                         sizeof(struct dev_match_result));
1975
1976                 /*
1977                  * If we don't have enough space to put in another
1978                  * match result, save our position and tell the
1979                  * user there are more devices to check.
1980                  */
1981                 if (spaceleft < sizeof(struct dev_match_result)) {
1982                         struct periph_driver **pdrv;
1983
1984                         pdrv = NULL;
1985                         bzero(&cdm->pos, sizeof(cdm->pos));
1986                         cdm->pos.position_type =
1987                                 CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
1988                                 CAM_DEV_POS_PERIPH;
1989
1990                         /*
1991                          * This may look a bit non-sensical, but it is
1992                          * actually quite logical.  There are very few
1993                          * peripheral drivers, and bloating every peripheral
1994                          * structure with a pointer back to its parent
1995                          * peripheral driver linker set entry would cost
1996                          * more in the long run than doing this quick lookup.
1997                          */
1998                         for (pdrv = periph_drivers; *pdrv != NULL; pdrv++) {
1999                                 if (strcmp((*pdrv)->driver_name,
2000                                     periph->periph_name) == 0)
2001                                         break;
2002                         }
2003
2004                         if (*pdrv == NULL) {
2005                                 cdm->status = CAM_DEV_MATCH_ERROR;
2006                                 return(0);
2007                         }
2008
2009                         cdm->pos.cookie.pdrv = pdrv;
2010                         /*
2011                          * The periph generation slot does double duty, as
2012                          * does the periph pointer slot.  They are used for
2013                          * both edt and pdrv lookups and positioning.
2014                          */
2015                         cdm->pos.cookie.periph = periph;
2016                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
2017                                 (*pdrv)->generation;
2018                         cdm->status = CAM_DEV_MATCH_MORE;
2019                         return(0);
2020                 }
2021
2022                 j = cdm->num_matches;
2023                 cdm->num_matches++;
2024                 cdm->matches[j].type = DEV_MATCH_PERIPH;
2025                 cdm->matches[j].result.periph_result.path_id =
2026                         periph->path->bus->path_id;
2027
2028                 /*
2029                  * The transport layer peripheral doesn't have a target or
2030                  * lun.
2031                  */
2032                 if (periph->path->target)
2033                         cdm->matches[j].result.periph_result.target_id =
2034                                 periph->path->target->target_id;
2035                 else
2036                         cdm->matches[j].result.periph_result.target_id =
2037                                 CAM_TARGET_WILDCARD;
2038
2039                 if (periph->path->device)
2040                         cdm->matches[j].result.periph_result.target_lun =
2041                                 periph->path->device->lun_id;
2042                 else
2043                         cdm->matches[j].result.periph_result.target_lun =
2044                                 CAM_LUN_WILDCARD;
2045
2046                 cdm->matches[j].result.periph_result.unit_number =
2047                         periph->unit_number;
2048                 l = sizeof(cdm->matches[j].result.periph_result.periph_name);
2049                 strlcpy(cdm->matches[j].result.periph_result.periph_name,
2050                         periph->periph_name, l);
2051         }
2052
2053         return(1);
2054 }
2055
2056 static int
2057 xptperiphlistmatch(struct ccb_dev_match *cdm)
2058 {
2059         int ret;
2060
2061         cdm->num_matches = 0;
2062
2063         /*
2064          * At this point in the edt traversal function, we check the bus
2065          * list generation to make sure that no busses have been added or
2066          * removed since the user last sent a XPT_DEV_MATCH ccb through.
2067          * For the peripheral driver list traversal function, however, we
2068          * don't have to worry about new peripheral driver types coming or
2069          * going; they're in a linker set, and therefore can't change
2070          * without a recompile.
2071          */
2072
2073         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2074          && (cdm->pos.cookie.pdrv != NULL))
2075                 ret = xptpdrvtraverse(
2076                                 (struct periph_driver **)cdm->pos.cookie.pdrv,
2077                                 xptplistpdrvfunc, cdm);
2078         else
2079                 ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2080
2081         /*
2082          * If we get back 0, that means that we had to stop before fully
2083          * traversing the peripheral driver tree.  It also means that one of
2084          * the subroutines has set the status field to the proper value.  If
2085          * we get back 1, we've fully traversed the EDT and copied out any
2086          * matching entries.
2087          */
2088         if (ret == 1)
2089                 cdm->status = CAM_DEV_MATCH_LAST;
2090
2091         return(ret);
2092 }
2093
2094 static int
2095 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2096 {
2097         struct cam_eb *bus, *next_bus;
2098         int retval;
2099
2100         retval = 1;
2101         if (start_bus)
2102                 bus = start_bus;
2103         else {
2104                 xpt_lock_buses();
2105                 bus = TAILQ_FIRST(&xsoftc.xpt_busses);
2106                 if (bus == NULL) {
2107                         xpt_unlock_buses();
2108                         return (retval);
2109                 }
2110                 bus->refcount++;
2111                 xpt_unlock_buses();
2112         }
2113         for (; bus != NULL; bus = next_bus) {
2114                 retval = tr_func(bus, arg);
2115                 if (retval == 0) {
2116                         xpt_release_bus(bus);
2117                         break;
2118                 }
2119                 xpt_lock_buses();
2120                 next_bus = TAILQ_NEXT(bus, links);
2121                 if (next_bus)
2122                         next_bus->refcount++;
2123                 xpt_unlock_buses();
2124                 xpt_release_bus(bus);
2125         }
2126         return(retval);
2127 }
2128
2129 static int
2130 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2131                   xpt_targetfunc_t *tr_func, void *arg)
2132 {
2133         struct cam_et *target, *next_target;
2134         int retval;
2135
2136         retval = 1;
2137         if (start_target)
2138                 target = start_target;
2139         else {
2140                 mtx_lock(&bus->eb_mtx);
2141                 target = TAILQ_FIRST(&bus->et_entries);
2142                 if (target == NULL) {
2143                         mtx_unlock(&bus->eb_mtx);
2144                         return (retval);
2145                 }
2146                 target->refcount++;
2147                 mtx_unlock(&bus->eb_mtx);
2148         }
2149         for (; target != NULL; target = next_target) {
2150                 retval = tr_func(target, arg);
2151                 if (retval == 0) {
2152                         xpt_release_target(target);
2153                         break;
2154                 }
2155                 mtx_lock(&bus->eb_mtx);
2156                 next_target = TAILQ_NEXT(target, links);
2157                 if (next_target)
2158                         next_target->refcount++;
2159                 mtx_unlock(&bus->eb_mtx);
2160                 xpt_release_target(target);
2161         }
2162         return(retval);
2163 }
2164
2165 static int
2166 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2167                   xpt_devicefunc_t *tr_func, void *arg)
2168 {
2169         struct cam_eb *bus;
2170         struct cam_ed *device, *next_device;
2171         int retval;
2172
2173         retval = 1;
2174         bus = target->bus;
2175         if (start_device)
2176                 device = start_device;
2177         else {
2178                 mtx_lock(&bus->eb_mtx);
2179                 device = TAILQ_FIRST(&target->ed_entries);
2180                 if (device == NULL) {
2181                         mtx_unlock(&bus->eb_mtx);
2182                         return (retval);
2183                 }
2184                 device->refcount++;
2185                 mtx_unlock(&bus->eb_mtx);
2186         }
2187         for (; device != NULL; device = next_device) {
2188                 mtx_lock(&device->device_mtx);
2189                 retval = tr_func(device, arg);
2190                 mtx_unlock(&device->device_mtx);
2191                 if (retval == 0) {
2192                         xpt_release_device(device);
2193                         break;
2194                 }
2195                 mtx_lock(&bus->eb_mtx);
2196                 next_device = TAILQ_NEXT(device, links);
2197                 if (next_device)
2198                         next_device->refcount++;
2199                 mtx_unlock(&bus->eb_mtx);
2200                 xpt_release_device(device);
2201         }
2202         return(retval);
2203 }
2204
2205 static int
2206 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2207                   xpt_periphfunc_t *tr_func, void *arg)
2208 {
2209         struct cam_eb *bus;
2210         struct cam_periph *periph, *next_periph;
2211         int retval;
2212
2213         retval = 1;
2214
2215         bus = device->target->bus;
2216         if (start_periph)
2217                 periph = start_periph;
2218         else {
2219                 xpt_lock_buses();
2220                 mtx_lock(&bus->eb_mtx);
2221                 periph = SLIST_FIRST(&device->periphs);
2222                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2223                         periph = SLIST_NEXT(periph, periph_links);
2224                 if (periph == NULL) {
2225                         mtx_unlock(&bus->eb_mtx);
2226                         xpt_unlock_buses();
2227                         return (retval);
2228                 }
2229                 periph->refcount++;
2230                 mtx_unlock(&bus->eb_mtx);
2231                 xpt_unlock_buses();
2232         }
2233         for (; periph != NULL; periph = next_periph) {
2234                 retval = tr_func(periph, arg);
2235                 if (retval == 0) {
2236                         cam_periph_release_locked(periph);
2237                         break;
2238                 }
2239                 xpt_lock_buses();
2240                 mtx_lock(&bus->eb_mtx);
2241                 next_periph = SLIST_NEXT(periph, periph_links);
2242                 while (next_periph != NULL &&
2243                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2244                         next_periph = SLIST_NEXT(next_periph, periph_links);
2245                 if (next_periph)
2246                         next_periph->refcount++;
2247                 mtx_unlock(&bus->eb_mtx);
2248                 xpt_unlock_buses();
2249                 cam_periph_release_locked(periph);
2250         }
2251         return(retval);
2252 }
2253
2254 static int
2255 xptpdrvtraverse(struct periph_driver **start_pdrv,
2256                 xpt_pdrvfunc_t *tr_func, void *arg)
2257 {
2258         struct periph_driver **pdrv;
2259         int retval;
2260
2261         retval = 1;
2262
2263         /*
2264          * We don't traverse the peripheral driver list like we do the
2265          * other lists, because it is a linker set, and therefore cannot be
2266          * changed during runtime.  If the peripheral driver list is ever
2267          * re-done to be something other than a linker set (i.e. it can
2268          * change while the system is running), the list traversal should
2269          * be modified to work like the other traversal functions.
2270          */
2271         for (pdrv = (start_pdrv ? start_pdrv : periph_drivers);
2272              *pdrv != NULL; pdrv++) {
2273                 retval = tr_func(pdrv, arg);
2274
2275                 if (retval == 0)
2276                         return(retval);
2277         }
2278
2279         return(retval);
2280 }
2281
2282 static int
2283 xptpdperiphtraverse(struct periph_driver **pdrv,
2284                     struct cam_periph *start_periph,
2285                     xpt_periphfunc_t *tr_func, void *arg)
2286 {
2287         struct cam_periph *periph, *next_periph;
2288         int retval;
2289
2290         retval = 1;
2291
2292         if (start_periph)
2293                 periph = start_periph;
2294         else {
2295                 xpt_lock_buses();
2296                 periph = TAILQ_FIRST(&(*pdrv)->units);
2297                 while (periph != NULL && (periph->flags & CAM_PERIPH_FREE) != 0)
2298                         periph = TAILQ_NEXT(periph, unit_links);
2299                 if (periph == NULL) {
2300                         xpt_unlock_buses();
2301                         return (retval);
2302                 }
2303                 periph->refcount++;
2304                 xpt_unlock_buses();
2305         }
2306         for (; periph != NULL; periph = next_periph) {
2307                 cam_periph_lock(periph);
2308                 retval = tr_func(periph, arg);
2309                 cam_periph_unlock(periph);
2310                 if (retval == 0) {
2311                         cam_periph_release(periph);
2312                         break;
2313                 }
2314                 xpt_lock_buses();
2315                 next_periph = TAILQ_NEXT(periph, unit_links);
2316                 while (next_periph != NULL &&
2317                     (next_periph->flags & CAM_PERIPH_FREE) != 0)
2318                         next_periph = TAILQ_NEXT(next_periph, unit_links);
2319                 if (next_periph)
2320                         next_periph->refcount++;
2321                 xpt_unlock_buses();
2322                 cam_periph_release(periph);
2323         }
2324         return(retval);
2325 }
2326
2327 static int
2328 xptdefbusfunc(struct cam_eb *bus, void *arg)
2329 {
2330         struct xpt_traverse_config *tr_config;
2331
2332         tr_config = (struct xpt_traverse_config *)arg;
2333
2334         if (tr_config->depth == XPT_DEPTH_BUS) {
2335                 xpt_busfunc_t *tr_func;
2336
2337                 tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2338
2339                 return(tr_func(bus, tr_config->tr_arg));
2340         } else
2341                 return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2342 }
2343
2344 static int
2345 xptdeftargetfunc(struct cam_et *target, void *arg)
2346 {
2347         struct xpt_traverse_config *tr_config;
2348
2349         tr_config = (struct xpt_traverse_config *)arg;
2350
2351         if (tr_config->depth == XPT_DEPTH_TARGET) {
2352                 xpt_targetfunc_t *tr_func;
2353
2354                 tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2355
2356                 return(tr_func(target, tr_config->tr_arg));
2357         } else
2358                 return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2359 }
2360
2361 static int
2362 xptdefdevicefunc(struct cam_ed *device, void *arg)
2363 {
2364         struct xpt_traverse_config *tr_config;
2365
2366         tr_config = (struct xpt_traverse_config *)arg;
2367
2368         if (tr_config->depth == XPT_DEPTH_DEVICE) {
2369                 xpt_devicefunc_t *tr_func;
2370
2371                 tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2372
2373                 return(tr_func(device, tr_config->tr_arg));
2374         } else
2375                 return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2376 }
2377
2378 static int
2379 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2380 {
2381         struct xpt_traverse_config *tr_config;
2382         xpt_periphfunc_t *tr_func;
2383
2384         tr_config = (struct xpt_traverse_config *)arg;
2385
2386         tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2387
2388         /*
2389          * Unlike the other default functions, we don't check for depth
2390          * here.  The peripheral driver level is the last level in the EDT,
2391          * so if we're here, we should execute the function in question.
2392          */
2393         return(tr_func(periph, tr_config->tr_arg));
2394 }
2395
2396 /*
2397  * Execute the given function for every bus in the EDT.
2398  */
2399 static int
2400 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2401 {
2402         struct xpt_traverse_config tr_config;
2403
2404         tr_config.depth = XPT_DEPTH_BUS;
2405         tr_config.tr_func = tr_func;
2406         tr_config.tr_arg = arg;
2407
2408         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2409 }
2410
2411 /*
2412  * Execute the given function for every device in the EDT.
2413  */
2414 static int
2415 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2416 {
2417         struct xpt_traverse_config tr_config;
2418
2419         tr_config.depth = XPT_DEPTH_DEVICE;
2420         tr_config.tr_func = tr_func;
2421         tr_config.tr_arg = arg;
2422
2423         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2424 }
2425
2426 static int
2427 xptsetasyncfunc(struct cam_ed *device, void *arg)
2428 {
2429         struct cam_path path;
2430         struct ccb_getdev cgd;
2431         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2432
2433         /*
2434          * Don't report unconfigured devices (Wildcard devs,
2435          * devices only for target mode, device instances
2436          * that have been invalidated but are waiting for
2437          * their last reference count to be released).
2438          */
2439         if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2440                 return (1);
2441
2442         xpt_compile_path(&path,
2443                          NULL,
2444                          device->target->bus->path_id,
2445                          device->target->target_id,
2446                          device->lun_id);
2447         xpt_setup_ccb(&cgd.ccb_h, &path, CAM_PRIORITY_NORMAL);
2448         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2449         xpt_action((union ccb *)&cgd);
2450         csa->callback(csa->callback_arg,
2451                             AC_FOUND_DEVICE,
2452                             &path, &cgd);
2453         xpt_release_path(&path);
2454
2455         return(1);
2456 }
2457
2458 static int
2459 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2460 {
2461         struct cam_path path;
2462         struct ccb_pathinq cpi;
2463         struct ccb_setasync *csa = (struct ccb_setasync *)arg;
2464
2465         xpt_compile_path(&path, /*periph*/NULL,
2466                          bus->path_id,
2467                          CAM_TARGET_WILDCARD,
2468                          CAM_LUN_WILDCARD);
2469         xpt_path_lock(&path);
2470         xpt_setup_ccb(&cpi.ccb_h, &path, CAM_PRIORITY_NORMAL);
2471         cpi.ccb_h.func_code = XPT_PATH_INQ;
2472         xpt_action((union ccb *)&cpi);
2473         csa->callback(csa->callback_arg,
2474                             AC_PATH_REGISTERED,
2475                             &path, &cpi);
2476         xpt_path_unlock(&path);
2477         xpt_release_path(&path);
2478
2479         return(1);
2480 }
2481
2482 void
2483 xpt_action(union ccb *start_ccb)
2484 {
2485
2486         CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE,
2487             ("xpt_action: func %#x %s\n", start_ccb->ccb_h.func_code,
2488                 xpt_action_name(start_ccb->ccb_h.func_code)));
2489
2490         start_ccb->ccb_h.status = CAM_REQ_INPROG;
2491         (*(start_ccb->ccb_h.path->bus->xport->action))(start_ccb);
2492 }
2493
2494 void
2495 xpt_action_default(union ccb *start_ccb)
2496 {
2497         struct cam_path *path;
2498         struct cam_sim *sim;
2499         struct mtx *mtx;
2500
2501         path = start_ccb->ccb_h.path;
2502         CAM_DEBUG(path, CAM_DEBUG_TRACE,
2503             ("xpt_action_default: func %#x %s\n", start_ccb->ccb_h.func_code,
2504                 xpt_action_name(start_ccb->ccb_h.func_code)));
2505
2506         switch (start_ccb->ccb_h.func_code) {
2507         case XPT_SCSI_IO:
2508         {
2509                 struct cam_ed *device;
2510
2511                 /*
2512                  * For the sake of compatibility with SCSI-1
2513                  * devices that may not understand the identify
2514                  * message, we include lun information in the
2515                  * second byte of all commands.  SCSI-1 specifies
2516                  * that luns are a 3 bit value and reserves only 3
2517                  * bits for lun information in the CDB.  Later
2518                  * revisions of the SCSI spec allow for more than 8
2519                  * luns, but have deprecated lun information in the
2520                  * CDB.  So, if the lun won't fit, we must omit.
2521                  *
2522                  * Also be aware that during initial probing for devices,
2523                  * the inquiry information is unknown but initialized to 0.
2524                  * This means that this code will be exercised while probing
2525                  * devices with an ANSI revision greater than 2.
2526                  */
2527                 device = path->device;
2528                 if (device->protocol_version <= SCSI_REV_2
2529                  && start_ccb->ccb_h.target_lun < 8
2530                  && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2531
2532                         start_ccb->csio.cdb_io.cdb_bytes[1] |=
2533                             start_ccb->ccb_h.target_lun << 5;
2534                 }
2535                 start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2536         }
2537         /* FALLTHROUGH */
2538         case XPT_TARGET_IO:
2539         case XPT_CONT_TARGET_IO:
2540                 start_ccb->csio.sense_resid = 0;
2541                 start_ccb->csio.resid = 0;
2542                 /* FALLTHROUGH */
2543         case XPT_ATA_IO:
2544                 if (start_ccb->ccb_h.func_code == XPT_ATA_IO)
2545                         start_ccb->ataio.resid = 0;
2546                 /* FALLTHROUGH */
2547         case XPT_NVME_IO:
2548                 if (start_ccb->ccb_h.func_code == XPT_NVME_IO)
2549                         start_ccb->nvmeio.resid = 0;
2550                 /* FALLTHROUGH */
2551         case XPT_RESET_DEV:
2552         case XPT_ENG_EXEC:
2553         case XPT_SMP_IO:
2554         {
2555                 struct cam_devq *devq;
2556
2557                 devq = path->bus->sim->devq;
2558                 mtx_lock(&devq->send_mtx);
2559                 cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2560                 if (xpt_schedule_devq(devq, path->device) != 0)
2561                         xpt_run_devq(devq);
2562                 mtx_unlock(&devq->send_mtx);
2563                 break;
2564         }
2565         case XPT_CALC_GEOMETRY:
2566                 /* Filter out garbage */
2567                 if (start_ccb->ccg.block_size == 0
2568                  || start_ccb->ccg.volume_size == 0) {
2569                         start_ccb->ccg.cylinders = 0;
2570                         start_ccb->ccg.heads = 0;
2571                         start_ccb->ccg.secs_per_track = 0;
2572                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2573                         break;
2574                 }
2575 #if defined(PC98) || defined(__sparc64__)
2576                 /*
2577                  * In a PC-98 system, geometry translation depens on
2578                  * the "real" device geometry obtained from mode page 4.
2579                  * SCSI geometry translation is performed in the
2580                  * initialization routine of the SCSI BIOS and the result
2581                  * stored in host memory.  If the translation is available
2582                  * in host memory, use it.  If not, rely on the default
2583                  * translation the device driver performs.
2584                  * For sparc64, we may need adjust the geometry of large
2585                  * disks in order to fit the limitations of the 16-bit
2586                  * fields of the VTOC8 disk label.
2587                  */
2588                 if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2589                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2590                         break;
2591                 }
2592 #endif
2593                 goto call_sim;
2594         case XPT_ABORT:
2595         {
2596                 union ccb* abort_ccb;
2597
2598                 abort_ccb = start_ccb->cab.abort_ccb;
2599                 if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2600                         struct cam_ed *device;
2601                         struct cam_devq *devq;
2602
2603                         device = abort_ccb->ccb_h.path->device;
2604                         devq = device->sim->devq;
2605
2606                         mtx_lock(&devq->send_mtx);
2607                         if (abort_ccb->ccb_h.pinfo.index > 0) {
2608                                 cam_ccbq_remove_ccb(&device->ccbq, abort_ccb);
2609                                 abort_ccb->ccb_h.status =
2610                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2611                                 xpt_freeze_devq_device(device, 1);
2612                                 mtx_unlock(&devq->send_mtx);
2613                                 xpt_done(abort_ccb);
2614                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2615                                 break;
2616                         }
2617                         mtx_unlock(&devq->send_mtx);
2618
2619                         if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2620                          && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2621                                 /*
2622                                  * We've caught this ccb en route to
2623                                  * the SIM.  Flag it for abort and the
2624                                  * SIM will do so just before starting
2625                                  * real work on the CCB.
2626                                  */
2627                                 abort_ccb->ccb_h.status =
2628                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2629                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2630                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2631                                 break;
2632                         }
2633                 }
2634                 if (XPT_FC_IS_QUEUED(abort_ccb)
2635                  && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2636                         /*
2637                          * It's already completed but waiting
2638                          * for our SWI to get to it.
2639                          */
2640                         start_ccb->ccb_h.status = CAM_UA_ABORT;
2641                         break;
2642                 }
2643                 /*
2644                  * If we weren't able to take care of the abort request
2645                  * in the XPT, pass the request down to the SIM for processing.
2646                  */
2647         }
2648         /* FALLTHROUGH */
2649         case XPT_ACCEPT_TARGET_IO:
2650         case XPT_EN_LUN:
2651         case XPT_IMMED_NOTIFY:
2652         case XPT_NOTIFY_ACK:
2653         case XPT_RESET_BUS:
2654         case XPT_IMMEDIATE_NOTIFY:
2655         case XPT_NOTIFY_ACKNOWLEDGE:
2656         case XPT_GET_SIM_KNOB_OLD:
2657         case XPT_GET_SIM_KNOB:
2658         case XPT_SET_SIM_KNOB:
2659         case XPT_GET_TRAN_SETTINGS:
2660         case XPT_SET_TRAN_SETTINGS:
2661         case XPT_PATH_INQ:
2662 call_sim:
2663                 sim = path->bus->sim;
2664                 mtx = sim->mtx;
2665                 if (mtx && !mtx_owned(mtx))
2666                         mtx_lock(mtx);
2667                 else
2668                         mtx = NULL;
2669                 CAM_DEBUG(path, CAM_DEBUG_TRACE,
2670                     ("sim->sim_action: func=%#x\n", start_ccb->ccb_h.func_code));
2671                 (*(sim->sim_action))(sim, start_ccb);
2672                 CAM_DEBUG(path, CAM_DEBUG_TRACE,
2673                     ("sim->sim_action: status=%#x\n", start_ccb->ccb_h.status));
2674                 if (mtx)
2675                         mtx_unlock(mtx);
2676                 break;
2677         case XPT_PATH_STATS:
2678                 start_ccb->cpis.last_reset = path->bus->last_reset;
2679                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2680                 break;
2681         case XPT_GDEV_TYPE:
2682         {
2683                 struct cam_ed *dev;
2684
2685                 dev = path->device;
2686                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2687                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2688                 } else {
2689                         struct ccb_getdev *cgd;
2690
2691                         cgd = &start_ccb->cgd;
2692                         cgd->protocol = dev->protocol;
2693                         cgd->inq_data = dev->inq_data;
2694                         cgd->ident_data = dev->ident_data;
2695                         cgd->inq_flags = dev->inq_flags;
2696                         cgd->nvme_data = dev->nvme_data;
2697                         cgd->nvme_cdata = dev->nvme_cdata;
2698                         cgd->ccb_h.status = CAM_REQ_CMP;
2699                         cgd->serial_num_len = dev->serial_num_len;
2700                         if ((dev->serial_num_len > 0)
2701                          && (dev->serial_num != NULL))
2702                                 bcopy(dev->serial_num, cgd->serial_num,
2703                                       dev->serial_num_len);
2704                 }
2705                 break;
2706         }
2707         case XPT_GDEV_STATS:
2708         {
2709                 struct ccb_getdevstats *cgds = &start_ccb->cgds;
2710                 struct cam_ed *dev = path->device;
2711                 struct cam_eb *bus = path->bus;
2712                 struct cam_et *tar = path->target;
2713                 struct cam_devq *devq = bus->sim->devq;
2714
2715                 mtx_lock(&devq->send_mtx);
2716                 cgds->dev_openings = dev->ccbq.dev_openings;
2717                 cgds->dev_active = dev->ccbq.dev_active;
2718                 cgds->allocated = dev->ccbq.allocated;
2719                 cgds->queued = cam_ccbq_pending_ccb_count(&dev->ccbq);
2720                 cgds->held = cgds->allocated - cgds->dev_active - cgds->queued;
2721                 cgds->last_reset = tar->last_reset;
2722                 cgds->maxtags = dev->maxtags;
2723                 cgds->mintags = dev->mintags;
2724                 if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2725                         cgds->last_reset = bus->last_reset;
2726                 mtx_unlock(&devq->send_mtx);
2727                 cgds->ccb_h.status = CAM_REQ_CMP;
2728                 break;
2729         }
2730         case XPT_GDEVLIST:
2731         {
2732                 struct cam_periph       *nperiph;
2733                 struct periph_list      *periph_head;
2734                 struct ccb_getdevlist   *cgdl;
2735                 u_int                   i;
2736                 struct cam_ed           *device;
2737                 int                     found;
2738
2739
2740                 found = 0;
2741
2742                 /*
2743                  * Don't want anyone mucking with our data.
2744                  */
2745                 device = path->device;
2746                 periph_head = &device->periphs;
2747                 cgdl = &start_ccb->cgdl;
2748
2749                 /*
2750                  * Check and see if the list has changed since the user
2751                  * last requested a list member.  If so, tell them that the
2752                  * list has changed, and therefore they need to start over
2753                  * from the beginning.
2754                  */
2755                 if ((cgdl->index != 0) &&
2756                     (cgdl->generation != device->generation)) {
2757                         cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
2758                         break;
2759                 }
2760
2761                 /*
2762                  * Traverse the list of peripherals and attempt to find
2763                  * the requested peripheral.
2764                  */
2765                 for (nperiph = SLIST_FIRST(periph_head), i = 0;
2766                      (nperiph != NULL) && (i <= cgdl->index);
2767                      nperiph = SLIST_NEXT(nperiph, periph_links), i++) {
2768                         if (i == cgdl->index) {
2769                                 strlcpy(cgdl->periph_name,
2770                                         nperiph->periph_name,
2771                                         sizeof(cgdl->periph_name));
2772                                 cgdl->unit_number = nperiph->unit_number;
2773                                 found = 1;
2774                         }
2775                 }
2776                 if (found == 0) {
2777                         cgdl->status = CAM_GDEVLIST_ERROR;
2778                         break;
2779                 }
2780
2781                 if (nperiph == NULL)
2782                         cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
2783                 else
2784                         cgdl->status = CAM_GDEVLIST_MORE_DEVS;
2785
2786                 cgdl->index++;
2787                 cgdl->generation = device->generation;
2788
2789                 cgdl->ccb_h.status = CAM_REQ_CMP;
2790                 break;
2791         }
2792         case XPT_DEV_MATCH:
2793         {
2794                 dev_pos_type position_type;
2795                 struct ccb_dev_match *cdm;
2796
2797                 cdm = &start_ccb->cdm;
2798
2799                 /*
2800                  * There are two ways of getting at information in the EDT.
2801                  * The first way is via the primary EDT tree.  It starts
2802                  * with a list of busses, then a list of targets on a bus,
2803                  * then devices/luns on a target, and then peripherals on a
2804                  * device/lun.  The "other" way is by the peripheral driver
2805                  * lists.  The peripheral driver lists are organized by
2806                  * peripheral driver.  (obviously)  So it makes sense to
2807                  * use the peripheral driver list if the user is looking
2808                  * for something like "da1", or all "da" devices.  If the
2809                  * user is looking for something on a particular bus/target
2810                  * or lun, it's generally better to go through the EDT tree.
2811                  */
2812
2813                 if (cdm->pos.position_type != CAM_DEV_POS_NONE)
2814                         position_type = cdm->pos.position_type;
2815                 else {
2816                         u_int i;
2817
2818                         position_type = CAM_DEV_POS_NONE;
2819
2820                         for (i = 0; i < cdm->num_patterns; i++) {
2821                                 if ((cdm->patterns[i].type == DEV_MATCH_BUS)
2822                                  ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
2823                                         position_type = CAM_DEV_POS_EDT;
2824                                         break;
2825                                 }
2826                         }
2827
2828                         if (cdm->num_patterns == 0)
2829                                 position_type = CAM_DEV_POS_EDT;
2830                         else if (position_type == CAM_DEV_POS_NONE)
2831                                 position_type = CAM_DEV_POS_PDRV;
2832                 }
2833
2834                 switch(position_type & CAM_DEV_POS_TYPEMASK) {
2835                 case CAM_DEV_POS_EDT:
2836                         xptedtmatch(cdm);
2837                         break;
2838                 case CAM_DEV_POS_PDRV:
2839                         xptperiphlistmatch(cdm);
2840                         break;
2841                 default:
2842                         cdm->status = CAM_DEV_MATCH_ERROR;
2843                         break;
2844                 }
2845
2846                 if (cdm->status == CAM_DEV_MATCH_ERROR)
2847                         start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2848                 else
2849                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2850
2851                 break;
2852         }
2853         case XPT_SASYNC_CB:
2854         {
2855                 struct ccb_setasync *csa;
2856                 struct async_node *cur_entry;
2857                 struct async_list *async_head;
2858                 u_int32_t added;
2859
2860                 csa = &start_ccb->csa;
2861                 added = csa->event_enable;
2862                 async_head = &path->device->asyncs;
2863
2864                 /*
2865                  * If there is already an entry for us, simply
2866                  * update it.
2867                  */
2868                 cur_entry = SLIST_FIRST(async_head);
2869                 while (cur_entry != NULL) {
2870                         if ((cur_entry->callback_arg == csa->callback_arg)
2871                          && (cur_entry->callback == csa->callback))
2872                                 break;
2873                         cur_entry = SLIST_NEXT(cur_entry, links);
2874                 }
2875
2876                 if (cur_entry != NULL) {
2877                         /*
2878                          * If the request has no flags set,
2879                          * remove the entry.
2880                          */
2881                         added &= ~cur_entry->event_enable;
2882                         if (csa->event_enable == 0) {
2883                                 SLIST_REMOVE(async_head, cur_entry,
2884                                              async_node, links);
2885                                 xpt_release_device(path->device);
2886                                 free(cur_entry, M_CAMXPT);
2887                         } else {
2888                                 cur_entry->event_enable = csa->event_enable;
2889                         }
2890                         csa->event_enable = added;
2891                 } else {
2892                         cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
2893                                            M_NOWAIT);
2894                         if (cur_entry == NULL) {
2895                                 csa->ccb_h.status = CAM_RESRC_UNAVAIL;
2896                                 break;
2897                         }
2898                         cur_entry->event_enable = csa->event_enable;
2899                         cur_entry->event_lock = (path->bus->sim->mtx &&
2900                             mtx_owned(path->bus->sim->mtx)) ? 1 : 0;
2901                         cur_entry->callback_arg = csa->callback_arg;
2902                         cur_entry->callback = csa->callback;
2903                         SLIST_INSERT_HEAD(async_head, cur_entry, links);
2904                         xpt_acquire_device(path->device);
2905                 }
2906                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2907                 break;
2908         }
2909         case XPT_REL_SIMQ:
2910         {
2911                 struct ccb_relsim *crs;
2912                 struct cam_ed *dev;
2913
2914                 crs = &start_ccb->crs;
2915                 dev = path->device;
2916                 if (dev == NULL) {
2917
2918                         crs->ccb_h.status = CAM_DEV_NOT_THERE;
2919                         break;
2920                 }
2921
2922                 if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
2923
2924                         /* Don't ever go below one opening */
2925                         if (crs->openings > 0) {
2926                                 xpt_dev_ccbq_resize(path, crs->openings);
2927                                 if (bootverbose) {
2928                                         xpt_print(path,
2929                                             "number of openings is now %d\n",
2930                                             crs->openings);
2931                                 }
2932                         }
2933                 }
2934
2935                 mtx_lock(&dev->sim->devq->send_mtx);
2936                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
2937
2938                         if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
2939
2940                                 /*
2941                                  * Just extend the old timeout and decrement
2942                                  * the freeze count so that a single timeout
2943                                  * is sufficient for releasing the queue.
2944                                  */
2945                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2946                                 callout_stop(&dev->callout);
2947                         } else {
2948
2949                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2950                         }
2951
2952                         callout_reset_sbt(&dev->callout,
2953                             SBT_1MS * crs->release_timeout, 0,
2954                             xpt_release_devq_timeout, dev, 0);
2955
2956                         dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
2957
2958                 }
2959
2960                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
2961
2962                         if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
2963                                 /*
2964                                  * Decrement the freeze count so that a single
2965                                  * completion is still sufficient to unfreeze
2966                                  * the queue.
2967                                  */
2968                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2969                         } else {
2970
2971                                 dev->flags |= CAM_DEV_REL_ON_COMPLETE;
2972                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2973                         }
2974                 }
2975
2976                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
2977
2978                         if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
2979                          || (dev->ccbq.dev_active == 0)) {
2980
2981                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
2982                         } else {
2983
2984                                 dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
2985                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
2986                         }
2987                 }
2988                 mtx_unlock(&dev->sim->devq->send_mtx);
2989
2990                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0)
2991                         xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
2992                 start_ccb->crs.qfrozen_cnt = dev->ccbq.queue.qfrozen_cnt;
2993                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2994                 break;
2995         }
2996         case XPT_DEBUG: {
2997                 struct cam_path *oldpath;
2998
2999                 /* Check that all request bits are supported. */
3000                 if (start_ccb->cdbg.flags & ~(CAM_DEBUG_COMPILE)) {
3001                         start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
3002                         break;
3003                 }
3004
3005                 cam_dflags = CAM_DEBUG_NONE;
3006                 if (cam_dpath != NULL) {
3007                         oldpath = cam_dpath;
3008                         cam_dpath = NULL;
3009                         xpt_free_path(oldpath);
3010                 }
3011                 if (start_ccb->cdbg.flags != CAM_DEBUG_NONE) {
3012                         if (xpt_create_path(&cam_dpath, NULL,
3013                                             start_ccb->ccb_h.path_id,
3014                                             start_ccb->ccb_h.target_id,
3015                                             start_ccb->ccb_h.target_lun) !=
3016                                             CAM_REQ_CMP) {
3017                                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3018                         } else {
3019                                 cam_dflags = start_ccb->cdbg.flags;
3020                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3021                                 xpt_print(cam_dpath, "debugging flags now %x\n",
3022                                     cam_dflags);
3023                         }
3024                 } else
3025                         start_ccb->ccb_h.status = CAM_REQ_CMP;
3026                 break;
3027         }
3028         case XPT_NOOP:
3029                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
3030                         xpt_freeze_devq(path, 1);
3031                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3032                 break;
3033         case XPT_REPROBE_LUN:
3034                 xpt_async(AC_INQ_CHANGED, path, NULL);
3035                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3036                 xpt_done(start_ccb);
3037                 break;
3038         default:
3039         case XPT_SDEV_TYPE:
3040         case XPT_TERM_IO:
3041         case XPT_ENG_INQ:
3042                 /* XXX Implement */
3043                 xpt_print_path(start_ccb->ccb_h.path);
3044                 printf("%s: CCB type %#x %s not supported\n", __func__,
3045                     start_ccb->ccb_h.func_code,
3046                     xpt_action_name(start_ccb->ccb_h.func_code));
3047                 start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3048                 if (start_ccb->ccb_h.func_code & XPT_FC_DEV_QUEUED) {
3049                         xpt_done(start_ccb);
3050                 }
3051                 break;
3052         }
3053         CAM_DEBUG(path, CAM_DEBUG_TRACE,
3054             ("xpt_action_default: func= %#x %s status %#x\n",
3055                 start_ccb->ccb_h.func_code,
3056                 xpt_action_name(start_ccb->ccb_h.func_code),
3057                 start_ccb->ccb_h.status));
3058 }
3059
3060 void
3061 xpt_polled_action(union ccb *start_ccb)
3062 {
3063         u_int32_t timeout;
3064         struct    cam_sim *sim;
3065         struct    cam_devq *devq;
3066         struct    cam_ed *dev;
3067         struct mtx *mtx;
3068
3069         timeout = start_ccb->ccb_h.timeout * 10;
3070         sim = start_ccb->ccb_h.path->bus->sim;
3071         devq = sim->devq;
3072         mtx = sim->mtx;
3073         dev = start_ccb->ccb_h.path->device;
3074
3075         mtx_unlock(&dev->device_mtx);
3076
3077         /*
3078          * Steal an opening so that no other queued requests
3079          * can get it before us while we simulate interrupts.
3080          */
3081         mtx_lock(&devq->send_mtx);
3082         dev->ccbq.dev_openings--;
3083         while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0) &&
3084             (--timeout > 0)) {
3085                 mtx_unlock(&devq->send_mtx);
3086                 DELAY(100);
3087                 if (mtx)
3088                         mtx_lock(mtx);
3089                 (*(sim->sim_poll))(sim);
3090                 if (mtx)
3091                         mtx_unlock(mtx);
3092                 camisr_runqueue();
3093                 mtx_lock(&devq->send_mtx);
3094         }
3095         dev->ccbq.dev_openings++;
3096         mtx_unlock(&devq->send_mtx);
3097
3098         if (timeout != 0) {
3099                 xpt_action(start_ccb);
3100                 while(--timeout > 0) {
3101                         if (mtx)
3102                                 mtx_lock(mtx);
3103                         (*(sim->sim_poll))(sim);
3104                         if (mtx)
3105                                 mtx_unlock(mtx);
3106                         camisr_runqueue();
3107                         if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3108                             != CAM_REQ_INPROG)
3109                                 break;
3110                         DELAY(100);
3111                 }
3112                 if (timeout == 0) {
3113                         /*
3114                          * XXX Is it worth adding a sim_timeout entry
3115                          * point so we can attempt recovery?  If
3116                          * this is only used for dumps, I don't think
3117                          * it is.
3118                          */
3119                         start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3120                 }
3121         } else {
3122                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3123         }
3124
3125         mtx_lock(&dev->device_mtx);
3126 }
3127
3128 /*
3129  * Schedule a peripheral driver to receive a ccb when its
3130  * target device has space for more transactions.
3131  */
3132 void
3133 xpt_schedule(struct cam_periph *periph, u_int32_t new_priority)
3134 {
3135
3136         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3137         cam_periph_assert(periph, MA_OWNED);
3138         if (new_priority < periph->scheduled_priority) {
3139                 periph->scheduled_priority = new_priority;
3140                 xpt_run_allocq(periph, 0);
3141         }
3142 }
3143
3144
3145 /*
3146  * Schedule a device to run on a given queue.
3147  * If the device was inserted as a new entry on the queue,
3148  * return 1 meaning the device queue should be run. If we
3149  * were already queued, implying someone else has already
3150  * started the queue, return 0 so the caller doesn't attempt
3151  * to run the queue.
3152  */
3153 static int
3154 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3155                  u_int32_t new_priority)
3156 {
3157         int retval;
3158         u_int32_t old_priority;
3159
3160         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3161
3162         old_priority = pinfo->priority;
3163
3164         /*
3165          * Are we already queued?
3166          */
3167         if (pinfo->index != CAM_UNQUEUED_INDEX) {
3168                 /* Simply reorder based on new priority */
3169                 if (new_priority < old_priority) {
3170                         camq_change_priority(queue, pinfo->index,
3171                                              new_priority);
3172                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3173                                         ("changed priority to %d\n",
3174                                          new_priority));
3175                         retval = 1;
3176                 } else
3177                         retval = 0;
3178         } else {
3179                 /* New entry on the queue */
3180                 if (new_priority < old_priority)
3181                         pinfo->priority = new_priority;
3182
3183                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3184                                 ("Inserting onto queue\n"));
3185                 pinfo->generation = ++queue->generation;
3186                 camq_insert(queue, pinfo);
3187                 retval = 1;
3188         }
3189         return (retval);
3190 }
3191
3192 static void
3193 xpt_run_allocq_task(void *context, int pending)
3194 {
3195         struct cam_periph *periph = context;
3196
3197         cam_periph_lock(periph);
3198         periph->flags &= ~CAM_PERIPH_RUN_TASK;
3199         xpt_run_allocq(periph, 1);
3200         cam_periph_unlock(periph);
3201         cam_periph_release(periph);
3202 }
3203
3204 static void
3205 xpt_run_allocq(struct cam_periph *periph, int sleep)
3206 {
3207         struct cam_ed   *device;
3208         union ccb       *ccb;
3209         uint32_t         prio;
3210
3211         cam_periph_assert(periph, MA_OWNED);
3212         if (periph->periph_allocating)
3213                 return;
3214         periph->periph_allocating = 1;
3215         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_allocq(%p)\n", periph));
3216         device = periph->path->device;
3217         ccb = NULL;
3218 restart:
3219         while ((prio = min(periph->scheduled_priority,
3220             periph->immediate_priority)) != CAM_PRIORITY_NONE &&
3221             (periph->periph_allocated - (ccb != NULL ? 1 : 0) <
3222              device->ccbq.total_openings || prio <= CAM_PRIORITY_OOB)) {
3223
3224                 if (ccb == NULL &&
3225                     (ccb = xpt_get_ccb_nowait(periph)) == NULL) {
3226                         if (sleep) {
3227                                 ccb = xpt_get_ccb(periph);
3228                                 goto restart;
3229                         }
3230                         if (periph->flags & CAM_PERIPH_RUN_TASK)
3231                                 break;
3232                         cam_periph_doacquire(periph);
3233                         periph->flags |= CAM_PERIPH_RUN_TASK;
3234                         taskqueue_enqueue(xsoftc.xpt_taskq,
3235                             &periph->periph_run_task);
3236                         break;
3237                 }
3238                 xpt_setup_ccb(&ccb->ccb_h, periph->path, prio);
3239                 if (prio == periph->immediate_priority) {
3240                         periph->immediate_priority = CAM_PRIORITY_NONE;
3241                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3242                                         ("waking cam_periph_getccb()\n"));
3243                         SLIST_INSERT_HEAD(&periph->ccb_list, &ccb->ccb_h,
3244                                           periph_links.sle);
3245                         wakeup(&periph->ccb_list);
3246                 } else {
3247                         periph->scheduled_priority = CAM_PRIORITY_NONE;
3248                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3249                                         ("calling periph_start()\n"));
3250                         periph->periph_start(periph, ccb);
3251                 }
3252                 ccb = NULL;
3253         }
3254         if (ccb != NULL)
3255                 xpt_release_ccb(ccb);
3256         periph->periph_allocating = 0;
3257 }
3258
3259 static void
3260 xpt_run_devq(struct cam_devq *devq)
3261 {
3262         char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
3263         struct mtx *mtx;
3264
3265         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_devq\n"));
3266
3267         devq->send_queue.qfrozen_cnt++;
3268         while ((devq->send_queue.entries > 0)
3269             && (devq->send_openings > 0)
3270             && (devq->send_queue.qfrozen_cnt <= 1)) {
3271                 struct  cam_ed *device;
3272                 union ccb *work_ccb;
3273                 struct  cam_sim *sim;
3274
3275                 device = (struct cam_ed *)camq_remove(&devq->send_queue,
3276                                                            CAMQ_HEAD);
3277                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3278                                 ("running device %p\n", device));
3279
3280                 work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3281                 if (work_ccb == NULL) {
3282                         printf("device on run queue with no ccbs???\n");
3283                         continue;
3284                 }
3285
3286                 if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3287
3288                         mtx_lock(&xsoftc.xpt_highpower_lock);
3289                         if (xsoftc.num_highpower <= 0) {
3290                                 /*
3291                                  * We got a high power command, but we
3292                                  * don't have any available slots.  Freeze
3293                                  * the device queue until we have a slot
3294                                  * available.
3295                                  */
3296                                 xpt_freeze_devq_device(device, 1);
3297                                 STAILQ_INSERT_TAIL(&xsoftc.highpowerq, device,
3298                                                    highpowerq_entry);
3299
3300                                 mtx_unlock(&xsoftc.xpt_highpower_lock);
3301                                 continue;
3302                         } else {
3303                                 /*
3304                                  * Consume a high power slot while
3305                                  * this ccb runs.
3306                                  */
3307                                 xsoftc.num_highpower--;
3308                         }
3309                         mtx_unlock(&xsoftc.xpt_highpower_lock);
3310                 }
3311                 cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3312                 cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3313                 devq->send_openings--;
3314                 devq->send_active++;
3315                 xpt_schedule_devq(devq, device);
3316                 mtx_unlock(&devq->send_mtx);
3317
3318                 if ((work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0) {
3319                         /*
3320                          * The client wants to freeze the queue
3321                          * after this CCB is sent.
3322                          */
3323                         xpt_freeze_devq(work_ccb->ccb_h.path, 1);
3324                 }
3325
3326                 /* In Target mode, the peripheral driver knows best... */
3327                 if (work_ccb->ccb_h.func_code == XPT_SCSI_IO) {
3328                         if ((device->inq_flags & SID_CmdQue) != 0
3329                          && work_ccb->csio.tag_action != CAM_TAG_ACTION_NONE)
3330                                 work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3331                         else
3332                                 /*
3333                                  * Clear this in case of a retried CCB that
3334                                  * failed due to a rejected tag.
3335                                  */
3336                                 work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3337                 }
3338
3339                 switch (work_ccb->ccb_h.func_code) {
3340                 case XPT_SCSI_IO:
3341                         CAM_DEBUG(work_ccb->ccb_h.path,
3342                             CAM_DEBUG_CDB,("%s. CDB: %s\n",
3343                              scsi_op_desc(work_ccb->csio.cdb_io.cdb_bytes[0],
3344                                           &device->inq_data),
3345                              scsi_cdb_string(work_ccb->csio.cdb_io.cdb_bytes,
3346                                              cdb_str, sizeof(cdb_str))));
3347                         break;
3348                 case XPT_ATA_IO:
3349                         CAM_DEBUG(work_ccb->ccb_h.path,
3350                             CAM_DEBUG_CDB,("%s. ACB: %s\n",
3351                              ata_op_string(&work_ccb->ataio.cmd),
3352                              ata_cmd_string(&work_ccb->ataio.cmd,
3353                                             cdb_str, sizeof(cdb_str))));
3354                         break;
3355                 case XPT_NVME_IO:
3356                         CAM_DEBUG(work_ccb->ccb_h.path,
3357                             CAM_DEBUG_CDB,("%s. NCB: %s\n",
3358                              nvme_op_string(&work_ccb->nvmeio.cmd),
3359                              nvme_cmd_string(&work_ccb->nvmeio.cmd,
3360                                             cdb_str, sizeof(cdb_str))));
3361                         break;
3362                 default:
3363                         break;
3364                 }
3365
3366                 /*
3367                  * Device queues can be shared among multiple SIM instances
3368                  * that reside on different busses.  Use the SIM from the
3369                  * queued device, rather than the one from the calling bus.
3370                  */
3371                 sim = device->sim;
3372                 mtx = sim->mtx;
3373                 if (mtx && !mtx_owned(mtx))
3374                         mtx_lock(mtx);
3375                 else
3376                         mtx = NULL;
3377                 work_ccb->ccb_h.qos.sim_data = sbinuptime(); // xxx uintprt_t too small 32bit platforms
3378                 (*(sim->sim_action))(sim, work_ccb);
3379                 if (mtx)
3380                         mtx_unlock(mtx);
3381                 mtx_lock(&devq->send_mtx);
3382         }
3383         devq->send_queue.qfrozen_cnt--;
3384 }
3385
3386 /*
3387  * This function merges stuff from the slave ccb into the master ccb, while
3388  * keeping important fields in the master ccb constant.
3389  */
3390 void
3391 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3392 {
3393
3394         /*
3395          * Pull fields that are valid for peripheral drivers to set
3396          * into the master CCB along with the CCB "payload".
3397          */
3398         master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3399         master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3400         master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3401         master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3402         bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3403               sizeof(union ccb) - sizeof(struct ccb_hdr));
3404 }
3405
3406 void
3407 xpt_setup_ccb_flags(struct ccb_hdr *ccb_h, struct cam_path *path,
3408                     u_int32_t priority, u_int32_t flags)
3409 {
3410
3411         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3412         ccb_h->pinfo.priority = priority;
3413         ccb_h->path = path;
3414         ccb_h->path_id = path->bus->path_id;
3415         if (path->target)
3416                 ccb_h->target_id = path->target->target_id;
3417         else
3418                 ccb_h->target_id = CAM_TARGET_WILDCARD;
3419         if (path->device) {
3420                 ccb_h->target_lun = path->device->lun_id;
3421                 ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3422         } else {
3423                 ccb_h->target_lun = CAM_TARGET_WILDCARD;
3424         }
3425         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3426         ccb_h->flags = flags;
3427         ccb_h->xflags = 0;
3428 }
3429
3430 void
3431 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3432 {
3433         xpt_setup_ccb_flags(ccb_h, path, priority, /*flags*/ 0);
3434 }
3435
3436 /* Path manipulation functions */
3437 cam_status
3438 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3439                 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3440 {
3441         struct     cam_path *path;
3442         cam_status status;
3443
3444         path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3445
3446         if (path == NULL) {
3447                 status = CAM_RESRC_UNAVAIL;
3448                 return(status);
3449         }
3450         status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3451         if (status != CAM_REQ_CMP) {
3452                 free(path, M_CAMPATH);
3453                 path = NULL;
3454         }
3455         *new_path_ptr = path;
3456         return (status);
3457 }
3458
3459 cam_status
3460 xpt_create_path_unlocked(struct cam_path **new_path_ptr,
3461                          struct cam_periph *periph, path_id_t path_id,
3462                          target_id_t target_id, lun_id_t lun_id)
3463 {
3464
3465         return (xpt_create_path(new_path_ptr, periph, path_id, target_id,
3466             lun_id));
3467 }
3468
3469 cam_status
3470 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3471                  path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3472 {
3473         struct       cam_eb *bus;
3474         struct       cam_et *target;
3475         struct       cam_ed *device;
3476         cam_status   status;
3477
3478         status = CAM_REQ_CMP;   /* Completed without error */
3479         target = NULL;          /* Wildcarded */
3480         device = NULL;          /* Wildcarded */
3481
3482         /*
3483          * We will potentially modify the EDT, so block interrupts
3484          * that may attempt to create cam paths.
3485          */
3486         bus = xpt_find_bus(path_id);
3487         if (bus == NULL) {
3488                 status = CAM_PATH_INVALID;
3489         } else {
3490                 xpt_lock_buses();
3491                 mtx_lock(&bus->eb_mtx);
3492                 target = xpt_find_target(bus, target_id);
3493                 if (target == NULL) {
3494                         /* Create one */
3495                         struct cam_et *new_target;
3496
3497                         new_target = xpt_alloc_target(bus, target_id);
3498                         if (new_target == NULL) {
3499                                 status = CAM_RESRC_UNAVAIL;
3500                         } else {
3501                                 target = new_target;
3502                         }
3503                 }
3504                 xpt_unlock_buses();
3505                 if (target != NULL) {
3506                         device = xpt_find_device(target, lun_id);
3507                         if (device == NULL) {
3508                                 /* Create one */
3509                                 struct cam_ed *new_device;
3510
3511                                 new_device =
3512                                     (*(bus->xport->alloc_device))(bus,
3513                                                                       target,
3514                                                                       lun_id);
3515                                 if (new_device == NULL) {
3516                                         status = CAM_RESRC_UNAVAIL;
3517                                 } else {
3518                                         device = new_device;
3519                                 }
3520                         }
3521                 }
3522                 mtx_unlock(&bus->eb_mtx);
3523         }
3524
3525         /*
3526          * Only touch the user's data if we are successful.
3527          */
3528         if (status == CAM_REQ_CMP) {
3529                 new_path->periph = perph;
3530                 new_path->bus = bus;
3531                 new_path->target = target;
3532                 new_path->device = device;
3533                 CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3534         } else {
3535                 if (device != NULL)
3536                         xpt_release_device(device);
3537                 if (target != NULL)
3538                         xpt_release_target(target);
3539                 if (bus != NULL)
3540                         xpt_release_bus(bus);
3541         }
3542         return (status);
3543 }
3544
3545 cam_status
3546 xpt_clone_path(struct cam_path **new_path_ptr, struct cam_path *path)
3547 {
3548         struct     cam_path *new_path;
3549
3550         new_path = (struct cam_path *)malloc(sizeof(*path), M_CAMPATH, M_NOWAIT);
3551         if (new_path == NULL)
3552                 return(CAM_RESRC_UNAVAIL);
3553         xpt_copy_path(new_path, path);
3554         *new_path_ptr = new_path;
3555         return (CAM_REQ_CMP);
3556 }
3557
3558 void
3559 xpt_copy_path(struct cam_path *new_path, struct cam_path *path)
3560 {
3561
3562         *new_path = *path;
3563         if (path->bus != NULL)
3564                 xpt_acquire_bus(path->bus);
3565         if (path->target != NULL)
3566                 xpt_acquire_target(path->target);
3567         if (path->device != NULL)
3568                 xpt_acquire_device(path->device);
3569 }
3570
3571 void
3572 xpt_release_path(struct cam_path *path)
3573 {
3574         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3575         if (path->device != NULL) {
3576                 xpt_release_device(path->device);
3577                 path->device = NULL;
3578         }
3579         if (path->target != NULL) {
3580                 xpt_release_target(path->target);
3581                 path->target = NULL;
3582         }
3583         if (path->bus != NULL) {
3584                 xpt_release_bus(path->bus);
3585                 path->bus = NULL;
3586         }
3587 }
3588
3589 void
3590 xpt_free_path(struct cam_path *path)
3591 {
3592
3593         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3594         xpt_release_path(path);
3595         free(path, M_CAMPATH);
3596 }
3597
3598 void
3599 xpt_path_counts(struct cam_path *path, uint32_t *bus_ref,
3600     uint32_t *periph_ref, uint32_t *target_ref, uint32_t *device_ref)
3601 {
3602
3603         xpt_lock_buses();
3604         if (bus_ref) {
3605                 if (path->bus)
3606                         *bus_ref = path->bus->refcount;
3607                 else
3608                         *bus_ref = 0;
3609         }
3610         if (periph_ref) {
3611                 if (path->periph)
3612                         *periph_ref = path->periph->refcount;
3613                 else
3614                         *periph_ref = 0;
3615         }
3616         xpt_unlock_buses();
3617         if (target_ref) {
3618                 if (path->target)
3619                         *target_ref = path->target->refcount;
3620                 else
3621                         *target_ref = 0;
3622         }
3623         if (device_ref) {
3624                 if (path->device)
3625                         *device_ref = path->device->refcount;
3626                 else
3627                         *device_ref = 0;
3628         }
3629 }
3630
3631 /*
3632  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3633  * in path1, 2 for match with wildcards in path2.
3634  */
3635 int
3636 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3637 {
3638         int retval = 0;
3639
3640         if (path1->bus != path2->bus) {
3641                 if (path1->bus->path_id == CAM_BUS_WILDCARD)
3642                         retval = 1;
3643                 else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3644                         retval = 2;
3645                 else
3646                         return (-1);
3647         }
3648         if (path1->target != path2->target) {
3649                 if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3650                         if (retval == 0)
3651                                 retval = 1;
3652                 } else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3653                         retval = 2;
3654                 else
3655                         return (-1);
3656         }
3657         if (path1->device != path2->device) {
3658                 if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3659                         if (retval == 0)
3660                                 retval = 1;
3661                 } else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3662                         retval = 2;
3663                 else
3664                         return (-1);
3665         }
3666         return (retval);
3667 }
3668
3669 int
3670 xpt_path_comp_dev(struct cam_path *path, struct cam_ed *dev)
3671 {
3672         int retval = 0;
3673
3674         if (path->bus != dev->target->bus) {
3675                 if (path->bus->path_id == CAM_BUS_WILDCARD)
3676                         retval = 1;
3677                 else if (dev->target->bus->path_id == CAM_BUS_WILDCARD)
3678                         retval = 2;
3679                 else
3680                         return (-1);
3681         }
3682         if (path->target != dev->target) {
3683                 if (path->target->target_id == CAM_TARGET_WILDCARD) {
3684                         if (retval == 0)
3685                                 retval = 1;
3686                 } else if (dev->target->target_id == CAM_TARGET_WILDCARD)
3687                         retval = 2;
3688                 else
3689                         return (-1);
3690         }
3691         if (path->device != dev) {
3692                 if (path->device->lun_id == CAM_LUN_WILDCARD) {
3693                         if (retval == 0)
3694                                 retval = 1;
3695                 } else if (dev->lun_id == CAM_LUN_WILDCARD)
3696                         retval = 2;
3697                 else
3698                         return (-1);
3699         }
3700         return (retval);
3701 }
3702
3703 void
3704 xpt_print_path(struct cam_path *path)
3705 {
3706
3707         if (path == NULL)
3708                 printf("(nopath): ");
3709         else {
3710                 if (path->periph != NULL)
3711                         printf("(%s%d:", path->periph->periph_name,
3712                                path->periph->unit_number);
3713                 else
3714                         printf("(noperiph:");
3715
3716                 if (path->bus != NULL)
3717                         printf("%s%d:%d:", path->bus->sim->sim_name,
3718                                path->bus->sim->unit_number,
3719                                path->bus->sim->bus_id);
3720                 else
3721                         printf("nobus:");
3722
3723                 if (path->target != NULL)
3724                         printf("%d:", path->target->target_id);
3725                 else
3726                         printf("X:");
3727
3728                 if (path->device != NULL)
3729                         printf("%jx): ", (uintmax_t)path->device->lun_id);
3730                 else
3731                         printf("X): ");
3732         }
3733 }
3734
3735 void
3736 xpt_print_device(struct cam_ed *device)
3737 {
3738
3739         if (device == NULL)
3740                 printf("(nopath): ");
3741         else {
3742                 printf("(noperiph:%s%d:%d:%d:%jx): ", device->sim->sim_name,
3743                        device->sim->unit_number,
3744                        device->sim->bus_id,
3745                        device->target->target_id,
3746                        (uintmax_t)device->lun_id);
3747         }
3748 }
3749
3750 void
3751 xpt_print(struct cam_path *path, const char *fmt, ...)
3752 {
3753         va_list ap;
3754         xpt_print_path(path);
3755         va_start(ap, fmt);
3756         vprintf(fmt, ap);
3757         va_end(ap);
3758 }
3759
3760 int
3761 xpt_path_string(struct cam_path *path, char *str, size_t str_len)
3762 {
3763         struct sbuf sb;
3764
3765         sbuf_new(&sb, str, str_len, 0);
3766
3767         if (path == NULL)
3768                 sbuf_printf(&sb, "(nopath): ");
3769         else {
3770                 if (path->periph != NULL)
3771                         sbuf_printf(&sb, "(%s%d:", path->periph->periph_name,
3772                                     path->periph->unit_number);
3773                 else
3774                         sbuf_printf(&sb, "(noperiph:");
3775
3776                 if (path->bus != NULL)
3777                         sbuf_printf(&sb, "%s%d:%d:", path->bus->sim->sim_name,
3778                                     path->bus->sim->unit_number,
3779                                     path->bus->sim->bus_id);
3780                 else
3781                         sbuf_printf(&sb, "nobus:");
3782
3783                 if (path->target != NULL)
3784                         sbuf_printf(&sb, "%d:", path->target->target_id);
3785                 else
3786                         sbuf_printf(&sb, "X:");
3787
3788                 if (path->device != NULL)
3789                         sbuf_printf(&sb, "%jx): ",
3790                             (uintmax_t)path->device->lun_id);
3791                 else
3792                         sbuf_printf(&sb, "X): ");
3793         }
3794         sbuf_finish(&sb);
3795
3796         return(sbuf_len(&sb));
3797 }
3798
3799 path_id_t
3800 xpt_path_path_id(struct cam_path *path)
3801 {
3802         return(path->bus->path_id);
3803 }
3804
3805 target_id_t
3806 xpt_path_target_id(struct cam_path *path)
3807 {
3808         if (path->target != NULL)
3809                 return (path->target->target_id);
3810         else
3811                 return (CAM_TARGET_WILDCARD);
3812 }
3813
3814 lun_id_t
3815 xpt_path_lun_id(struct cam_path *path)
3816 {
3817         if (path->device != NULL)
3818                 return (path->device->lun_id);
3819         else
3820                 return (CAM_LUN_WILDCARD);
3821 }
3822
3823 struct cam_sim *
3824 xpt_path_sim(struct cam_path *path)
3825 {
3826
3827         return (path->bus->sim);
3828 }
3829
3830 struct cam_periph*
3831 xpt_path_periph(struct cam_path *path)
3832 {
3833
3834         return (path->periph);
3835 }
3836
3837 /*
3838  * Release a CAM control block for the caller.  Remit the cost of the structure
3839  * to the device referenced by the path.  If the this device had no 'credits'
3840  * and peripheral drivers have registered async callbacks for this notification
3841  * call them now.
3842  */
3843 void
3844 xpt_release_ccb(union ccb *free_ccb)
3845 {
3846         struct   cam_ed *device;
3847         struct   cam_periph *periph;
3848
3849         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3850         xpt_path_assert(free_ccb->ccb_h.path, MA_OWNED);
3851         device = free_ccb->ccb_h.path->device;
3852         periph = free_ccb->ccb_h.path->periph;
3853
3854         xpt_free_ccb(free_ccb);
3855         periph->periph_allocated--;
3856         cam_ccbq_release_opening(&device->ccbq);
3857         xpt_run_allocq(periph, 0);
3858 }
3859
3860 /* Functions accessed by SIM drivers */
3861
3862 static struct xpt_xport xport_default = {
3863         .alloc_device = xpt_alloc_device_default,
3864         .action = xpt_action_default,
3865         .async = xpt_dev_async_default,
3866 };
3867
3868 /*
3869  * A sim structure, listing the SIM entry points and instance
3870  * identification info is passed to xpt_bus_register to hook the SIM
3871  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
3872  * for this new bus and places it in the array of busses and assigns
3873  * it a path_id.  The path_id may be influenced by "hard wiring"
3874  * information specified by the user.  Once interrupt services are
3875  * available, the bus will be probed.
3876  */
3877 int32_t
3878 xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
3879 {
3880         struct cam_eb *new_bus;
3881         struct cam_eb *old_bus;
3882         struct ccb_pathinq cpi;
3883         struct cam_path *path;
3884         cam_status status;
3885
3886         sim->bus_id = bus;
3887         new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
3888                                           M_CAMXPT, M_NOWAIT|M_ZERO);
3889         if (new_bus == NULL) {
3890                 /* Couldn't satisfy request */
3891                 return (CAM_RESRC_UNAVAIL);
3892         }
3893
3894         mtx_init(&new_bus->eb_mtx, "CAM bus lock", NULL, MTX_DEF);
3895         TAILQ_INIT(&new_bus->et_entries);
3896         cam_sim_hold(sim);
3897         new_bus->sim = sim;
3898         timevalclear(&new_bus->last_reset);
3899         new_bus->flags = 0;
3900         new_bus->refcount = 1;  /* Held until a bus_deregister event */
3901         new_bus->generation = 0;
3902
3903         xpt_lock_buses();
3904         sim->path_id = new_bus->path_id =
3905             xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
3906         old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
3907         while (old_bus != NULL
3908             && old_bus->path_id < new_bus->path_id)
3909                 old_bus = TAILQ_NEXT(old_bus, links);
3910         if (old_bus != NULL)
3911                 TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
3912         else
3913                 TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
3914         xsoftc.bus_generation++;
3915         xpt_unlock_buses();
3916
3917         /*
3918          * Set a default transport so that a PATH_INQ can be issued to
3919          * the SIM.  This will then allow for probing and attaching of
3920          * a more appropriate transport.
3921          */
3922         new_bus->xport = &xport_default;
3923
3924         status = xpt_create_path(&path, /*periph*/NULL, sim->path_id,
3925                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3926         if (status != CAM_REQ_CMP) {
3927                 xpt_release_bus(new_bus);
3928                 return (CAM_RESRC_UNAVAIL);
3929         }
3930
3931         xpt_setup_ccb(&cpi.ccb_h, path, CAM_PRIORITY_NORMAL);
3932         cpi.ccb_h.func_code = XPT_PATH_INQ;
3933         xpt_action((union ccb *)&cpi);
3934
3935         if (cpi.ccb_h.status == CAM_REQ_CMP) {
3936                 switch (cpi.transport) {
3937                 case XPORT_SPI:
3938                 case XPORT_SAS:
3939                 case XPORT_FC:
3940                 case XPORT_USB:
3941                 case XPORT_ISCSI:
3942                 case XPORT_SRP:
3943                 case XPORT_PPB:
3944                         new_bus->xport = scsi_get_xport();
3945                         break;
3946                 case XPORT_ATA:
3947                 case XPORT_SATA:
3948                         new_bus->xport = ata_get_xport();
3949                         break;
3950                 case XPORT_NVME:
3951                         new_bus->xport = nvme_get_xport();
3952                         break;
3953                 default:
3954                         new_bus->xport = &xport_default;
3955                         break;
3956                 }
3957         }
3958
3959         /* Notify interested parties */
3960         if (sim->path_id != CAM_XPT_PATH_ID) {
3961
3962                 xpt_async(AC_PATH_REGISTERED, path, &cpi);
3963                 if ((cpi.hba_misc & PIM_NOSCAN) == 0) {
3964                         union   ccb *scan_ccb;
3965
3966                         /* Initiate bus rescan. */
3967                         scan_ccb = xpt_alloc_ccb_nowait();
3968                         if (scan_ccb != NULL) {
3969                                 scan_ccb->ccb_h.path = path;
3970                                 scan_ccb->ccb_h.func_code = XPT_SCAN_BUS;
3971                                 scan_ccb->crcn.flags = 0;
3972                                 xpt_rescan(scan_ccb);
3973                         } else {
3974                                 xpt_print(path,
3975                                           "Can't allocate CCB to scan bus\n");
3976                                 xpt_free_path(path);
3977                         }
3978                 } else
3979                         xpt_free_path(path);
3980         } else
3981                 xpt_free_path(path);
3982         return (CAM_SUCCESS);
3983 }
3984
3985 int32_t
3986 xpt_bus_deregister(path_id_t pathid)
3987 {
3988         struct cam_path bus_path;
3989         cam_status status;
3990
3991         status = xpt_compile_path(&bus_path, NULL, pathid,
3992                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
3993         if (status != CAM_REQ_CMP)
3994                 return (status);
3995
3996         xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
3997         xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
3998
3999         /* Release the reference count held while registered. */
4000         xpt_release_bus(bus_path.bus);
4001         xpt_release_path(&bus_path);
4002
4003         return (CAM_REQ_CMP);
4004 }
4005
4006 static path_id_t
4007 xptnextfreepathid(void)
4008 {
4009         struct cam_eb *bus;
4010         path_id_t pathid;
4011         const char *strval;
4012
4013         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4014         pathid = 0;
4015         bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4016 retry:
4017         /* Find an unoccupied pathid */
4018         while (bus != NULL && bus->path_id <= pathid) {
4019                 if (bus->path_id == pathid)
4020                         pathid++;
4021                 bus = TAILQ_NEXT(bus, links);
4022         }
4023
4024         /*
4025          * Ensure that this pathid is not reserved for
4026          * a bus that may be registered in the future.
4027          */
4028         if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4029                 ++pathid;
4030                 /* Start the search over */
4031                 goto retry;
4032         }
4033         return (pathid);
4034 }
4035
4036 static path_id_t
4037 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4038 {
4039         path_id_t pathid;
4040         int i, dunit, val;
4041         char buf[32];
4042         const char *dname;
4043
4044         pathid = CAM_XPT_PATH_ID;
4045         snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4046         if (strcmp(buf, "xpt0") == 0 && sim_bus == 0)
4047                 return (pathid);
4048         i = 0;
4049         while ((resource_find_match(&i, &dname, &dunit, "at", buf)) == 0) {
4050                 if (strcmp(dname, "scbus")) {
4051                         /* Avoid a bit of foot shooting. */
4052                         continue;
4053                 }
4054                 if (dunit < 0)          /* unwired?! */
4055                         continue;
4056                 if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4057                         if (sim_bus == val) {
4058                                 pathid = dunit;
4059                                 break;
4060                         }
4061                 } else if (sim_bus == 0) {
4062                         /* Unspecified matches bus 0 */
4063                         pathid = dunit;
4064                         break;
4065                 } else {
4066                         printf("Ambiguous scbus configuration for %s%d "
4067                                "bus %d, cannot wire down.  The kernel "
4068                                "config entry for scbus%d should "
4069                                "specify a controller bus.\n"
4070                                "Scbus will be assigned dynamically.\n",
4071                                sim_name, sim_unit, sim_bus, dunit);
4072                         break;
4073                 }
4074         }
4075
4076         if (pathid == CAM_XPT_PATH_ID)
4077                 pathid = xptnextfreepathid();
4078         return (pathid);
4079 }
4080
4081 static const char *
4082 xpt_async_string(u_int32_t async_code)
4083 {
4084
4085         switch (async_code) {
4086         case AC_BUS_RESET: return ("AC_BUS_RESET");
4087         case AC_UNSOL_RESEL: return ("AC_UNSOL_RESEL");
4088         case AC_SCSI_AEN: return ("AC_SCSI_AEN");
4089         case AC_SENT_BDR: return ("AC_SENT_BDR");
4090         case AC_PATH_REGISTERED: return ("AC_PATH_REGISTERED");
4091         case AC_PATH_DEREGISTERED: return ("AC_PATH_DEREGISTERED");
4092         case AC_FOUND_DEVICE: return ("AC_FOUND_DEVICE");
4093         case AC_LOST_DEVICE: return ("AC_LOST_DEVICE");
4094         case AC_TRANSFER_NEG: return ("AC_TRANSFER_NEG");
4095         case AC_INQ_CHANGED: return ("AC_INQ_CHANGED");
4096         case AC_GETDEV_CHANGED: return ("AC_GETDEV_CHANGED");
4097         case AC_CONTRACT: return ("AC_CONTRACT");
4098         case AC_ADVINFO_CHANGED: return ("AC_ADVINFO_CHANGED");
4099         case AC_UNIT_ATTENTION: return ("AC_UNIT_ATTENTION");
4100         }
4101         return ("AC_UNKNOWN");
4102 }
4103
4104 static int
4105 xpt_async_size(u_int32_t async_code)
4106 {
4107
4108         switch (async_code) {
4109         case AC_BUS_RESET: return (0);
4110         case AC_UNSOL_RESEL: return (0);
4111         case AC_SCSI_AEN: return (0);
4112         case AC_SENT_BDR: return (0);
4113         case AC_PATH_REGISTERED: return (sizeof(struct ccb_pathinq));
4114         case AC_PATH_DEREGISTERED: return (0);
4115         case AC_FOUND_DEVICE: return (sizeof(struct ccb_getdev));
4116         case AC_LOST_DEVICE: return (0);
4117         case AC_TRANSFER_NEG: return (sizeof(struct ccb_trans_settings));
4118         case AC_INQ_CHANGED: return (0);
4119         case AC_GETDEV_CHANGED: return (0);
4120         case AC_CONTRACT: return (sizeof(struct ac_contract));
4121         case AC_ADVINFO_CHANGED: return (-1);
4122         case AC_UNIT_ATTENTION: return (sizeof(struct ccb_scsiio));
4123         }
4124         return (0);
4125 }
4126
4127 static int
4128 xpt_async_process_dev(struct cam_ed *device, void *arg)
4129 {
4130         union ccb *ccb = arg;
4131         struct cam_path *path = ccb->ccb_h.path;
4132         void *async_arg = ccb->casync.async_arg_ptr;
4133         u_int32_t async_code = ccb->casync.async_code;
4134         int relock;
4135
4136         if (path->device != device
4137          && path->device->lun_id != CAM_LUN_WILDCARD
4138          && device->lun_id != CAM_LUN_WILDCARD)
4139                 return (1);
4140
4141         /*
4142          * The async callback could free the device.
4143          * If it is a broadcast async, it doesn't hold
4144          * device reference, so take our own reference.
4145          */
4146         xpt_acquire_device(device);
4147
4148         /*
4149          * If async for specific device is to be delivered to
4150          * the wildcard client, take the specific device lock.
4151          * XXX: We may need a way for client to specify it.
4152          */
4153         if ((device->lun_id == CAM_LUN_WILDCARD &&
4154              path->device->lun_id != CAM_LUN_WILDCARD) ||
4155             (device->target->target_id == CAM_TARGET_WILDCARD &&
4156              path->target->target_id != CAM_TARGET_WILDCARD) ||
4157             (device->target->bus->path_id == CAM_BUS_WILDCARD &&
4158              path->target->bus->path_id != CAM_BUS_WILDCARD)) {
4159                 mtx_unlock(&device->device_mtx);
4160                 xpt_path_lock(path);
4161                 relock = 1;
4162         } else
4163                 relock = 0;
4164
4165         (*(device->target->bus->xport->async))(async_code,
4166             device->target->bus, device->target, device, async_arg);
4167         xpt_async_bcast(&device->asyncs, async_code, path, async_arg);
4168
4169         if (relock) {
4170                 xpt_path_unlock(path);
4171                 mtx_lock(&device->device_mtx);
4172         }
4173         xpt_release_device(device);
4174         return (1);
4175 }
4176
4177 static int
4178 xpt_async_process_tgt(struct cam_et *target, void *arg)
4179 {
4180         union ccb *ccb = arg;
4181         struct cam_path *path = ccb->ccb_h.path;
4182
4183         if (path->target != target
4184          && path->target->target_id != CAM_TARGET_WILDCARD
4185          && target->target_id != CAM_TARGET_WILDCARD)
4186                 return (1);
4187
4188         if (ccb->casync.async_code == AC_SENT_BDR) {
4189                 /* Update our notion of when the last reset occurred */
4190                 microtime(&target->last_reset);
4191         }
4192
4193         return (xptdevicetraverse(target, NULL, xpt_async_process_dev, ccb));
4194 }
4195
4196 static void
4197 xpt_async_process(struct cam_periph *periph, union ccb *ccb)
4198 {
4199         struct cam_eb *bus;
4200         struct cam_path *path;
4201         void *async_arg;
4202         u_int32_t async_code;
4203
4204         path = ccb->ccb_h.path;
4205         async_code = ccb->casync.async_code;
4206         async_arg = ccb->casync.async_arg_ptr;
4207         CAM_DEBUG(path, CAM_DEBUG_TRACE | CAM_DEBUG_INFO,
4208             ("xpt_async(%s)\n", xpt_async_string(async_code)));
4209         bus = path->bus;
4210
4211         if (async_code == AC_BUS_RESET) {
4212                 /* Update our notion of when the last reset occurred */
4213                 microtime(&bus->last_reset);
4214         }
4215
4216         xpttargettraverse(bus, NULL, xpt_async_process_tgt, ccb);
4217
4218         /*
4219          * If this wasn't a fully wildcarded async, tell all
4220          * clients that want all async events.
4221          */
4222         if (bus != xpt_periph->path->bus) {
4223                 xpt_path_lock(xpt_periph->path);
4224                 xpt_async_process_dev(xpt_periph->path->device, ccb);
4225                 xpt_path_unlock(xpt_periph->path);
4226         }
4227
4228         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4229                 xpt_release_devq(path, 1, TRUE);
4230         else
4231                 xpt_release_simq(path->bus->sim, TRUE);
4232         if (ccb->casync.async_arg_size > 0)
4233                 free(async_arg, M_CAMXPT);
4234         xpt_free_path(path);
4235         xpt_free_ccb(ccb);
4236 }
4237
4238 static void
4239 xpt_async_bcast(struct async_list *async_head,
4240                 u_int32_t async_code,
4241                 struct cam_path *path, void *async_arg)
4242 {
4243         struct async_node *cur_entry;
4244         struct mtx *mtx;
4245
4246         cur_entry = SLIST_FIRST(async_head);
4247         while (cur_entry != NULL) {
4248                 struct async_node *next_entry;
4249                 /*
4250                  * Grab the next list entry before we call the current
4251                  * entry's callback.  This is because the callback function
4252                  * can delete its async callback entry.
4253                  */
4254                 next_entry = SLIST_NEXT(cur_entry, links);
4255                 if ((cur_entry->event_enable & async_code) != 0) {
4256                         mtx = cur_entry->event_lock ?
4257                             path->device->sim->mtx : NULL;
4258                         if (mtx)
4259                                 mtx_lock(mtx);
4260                         cur_entry->callback(cur_entry->callback_arg,
4261                                             async_code, path,
4262                                             async_arg);
4263                         if (mtx)
4264                                 mtx_unlock(mtx);
4265                 }
4266                 cur_entry = next_entry;
4267         }
4268 }
4269
4270 void
4271 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4272 {
4273         union ccb *ccb;
4274         int size;
4275
4276         ccb = xpt_alloc_ccb_nowait();
4277         if (ccb == NULL) {
4278                 xpt_print(path, "Can't allocate CCB to send %s\n",
4279                     xpt_async_string(async_code));
4280                 return;
4281         }
4282
4283         if (xpt_clone_path(&ccb->ccb_h.path, path) != CAM_REQ_CMP) {
4284                 xpt_print(path, "Can't allocate path to send %s\n",
4285                     xpt_async_string(async_code));
4286                 xpt_free_ccb(ccb);
4287                 return;
4288         }
4289         ccb->ccb_h.path->periph = NULL;
4290         ccb->ccb_h.func_code = XPT_ASYNC;
4291         ccb->ccb_h.cbfcnp = xpt_async_process;
4292         ccb->ccb_h.flags |= CAM_UNLOCKED;
4293         ccb->casync.async_code = async_code;
4294         ccb->casync.async_arg_size = 0;
4295         size = xpt_async_size(async_code);
4296         CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
4297             ("xpt_async: func %#x %s aync_code %d %s\n",
4298                 ccb->ccb_h.func_code,
4299                 xpt_action_name(ccb->ccb_h.func_code),
4300                 async_code,
4301                 xpt_async_string(async_code)));
4302         if (size > 0 && async_arg != NULL) {
4303                 ccb->casync.async_arg_ptr = malloc(size, M_CAMXPT, M_NOWAIT);
4304                 if (ccb->casync.async_arg_ptr == NULL) {
4305                         xpt_print(path, "Can't allocate argument to send %s\n",
4306                             xpt_async_string(async_code));
4307                         xpt_free_path(ccb->ccb_h.path);
4308                         xpt_free_ccb(ccb);
4309                         return;
4310                 }
4311                 memcpy(ccb->casync.async_arg_ptr, async_arg, size);
4312                 ccb->casync.async_arg_size = size;
4313         } else if (size < 0) {
4314                 ccb->casync.async_arg_ptr = async_arg;
4315                 ccb->casync.async_arg_size = size;
4316         }
4317         if (path->device != NULL && path->device->lun_id != CAM_LUN_WILDCARD)
4318                 xpt_freeze_devq(path, 1);
4319         else
4320                 xpt_freeze_simq(path->bus->sim, 1);
4321         xpt_done(ccb);
4322 }
4323
4324 static void
4325 xpt_dev_async_default(u_int32_t async_code, struct cam_eb *bus,
4326                       struct cam_et *target, struct cam_ed *device,
4327                       void *async_arg)
4328 {
4329
4330         /*
4331          * We only need to handle events for real devices.
4332          */
4333         if (target->target_id == CAM_TARGET_WILDCARD
4334          || device->lun_id == CAM_LUN_WILDCARD)
4335                 return;
4336
4337         printf("%s called\n", __func__);
4338 }
4339
4340 static uint32_t
4341 xpt_freeze_devq_device(struct cam_ed *dev, u_int count)
4342 {
4343         struct cam_devq *devq;
4344         uint32_t freeze;
4345
4346         devq = dev->sim->devq;
4347         mtx_assert(&devq->send_mtx, MA_OWNED);
4348         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4349             ("xpt_freeze_devq_device(%d) %u->%u\n", count,
4350             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt + count));
4351         freeze = (dev->ccbq.queue.qfrozen_cnt += count);
4352         /* Remove frozen device from sendq. */
4353         if (device_is_queued(dev))
4354                 camq_remove(&devq->send_queue, dev->devq_entry.index);
4355         return (freeze);
4356 }
4357
4358 u_int32_t
4359 xpt_freeze_devq(struct cam_path *path, u_int count)
4360 {
4361         struct cam_ed   *dev = path->device;
4362         struct cam_devq *devq;
4363         uint32_t         freeze;
4364
4365         devq = dev->sim->devq;
4366         mtx_lock(&devq->send_mtx);
4367         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_freeze_devq(%d)\n", count));
4368         freeze = xpt_freeze_devq_device(dev, count);
4369         mtx_unlock(&devq->send_mtx);
4370         return (freeze);
4371 }
4372
4373 u_int32_t
4374 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4375 {
4376         struct cam_devq *devq;
4377         uint32_t         freeze;
4378
4379         devq = sim->devq;
4380         mtx_lock(&devq->send_mtx);
4381         freeze = (devq->send_queue.qfrozen_cnt += count);
4382         mtx_unlock(&devq->send_mtx);
4383         return (freeze);
4384 }
4385
4386 static void
4387 xpt_release_devq_timeout(void *arg)
4388 {
4389         struct cam_ed *dev;
4390         struct cam_devq *devq;
4391
4392         dev = (struct cam_ed *)arg;
4393         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE, ("xpt_release_devq_timeout\n"));
4394         devq = dev->sim->devq;
4395         mtx_assert(&devq->send_mtx, MA_OWNED);
4396         if (xpt_release_devq_device(dev, /*count*/1, /*run_queue*/TRUE))
4397                 xpt_run_devq(devq);
4398 }
4399
4400 void
4401 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4402 {
4403         struct cam_ed *dev;
4404         struct cam_devq *devq;
4405
4406         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_devq(%d, %d)\n",
4407             count, run_queue));
4408         dev = path->device;
4409         devq = dev->sim->devq;
4410         mtx_lock(&devq->send_mtx);
4411         if (xpt_release_devq_device(dev, count, run_queue))
4412                 xpt_run_devq(dev->sim->devq);
4413         mtx_unlock(&devq->send_mtx);
4414 }
4415
4416 static int
4417 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4418 {
4419
4420         mtx_assert(&dev->sim->devq->send_mtx, MA_OWNED);
4421         CAM_DEBUG_DEV(dev, CAM_DEBUG_TRACE,
4422             ("xpt_release_devq_device(%d, %d) %u->%u\n", count, run_queue,
4423             dev->ccbq.queue.qfrozen_cnt, dev->ccbq.queue.qfrozen_cnt - count));
4424         if (count > dev->ccbq.queue.qfrozen_cnt) {
4425 #ifdef INVARIANTS
4426                 printf("xpt_release_devq(): requested %u > present %u\n",
4427                     count, dev->ccbq.queue.qfrozen_cnt);
4428 #endif
4429                 count = dev->ccbq.queue.qfrozen_cnt;
4430         }
4431         dev->ccbq.queue.qfrozen_cnt -= count;
4432         if (dev->ccbq.queue.qfrozen_cnt == 0) {
4433                 /*
4434                  * No longer need to wait for a successful
4435                  * command completion.
4436                  */
4437                 dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4438                 /*
4439                  * Remove any timeouts that might be scheduled
4440                  * to release this queue.
4441                  */
4442                 if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4443                         callout_stop(&dev->callout);
4444                         dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4445                 }
4446                 /*
4447                  * Now that we are unfrozen schedule the
4448                  * device so any pending transactions are
4449                  * run.
4450                  */
4451                 xpt_schedule_devq(dev->sim->devq, dev);
4452         } else
4453                 run_queue = 0;
4454         return (run_queue);
4455 }
4456
4457 void
4458 xpt_release_simq(struct cam_sim *sim, int run_queue)
4459 {
4460         struct cam_devq *devq;
4461
4462         devq = sim->devq;
4463         mtx_lock(&devq->send_mtx);
4464         if (devq->send_queue.qfrozen_cnt <= 0) {
4465 #ifdef INVARIANTS
4466                 printf("xpt_release_simq: requested 1 > present %u\n",
4467                     devq->send_queue.qfrozen_cnt);
4468 #endif
4469         } else
4470                 devq->send_queue.qfrozen_cnt--;
4471         if (devq->send_queue.qfrozen_cnt == 0) {
4472                 /*
4473                  * If there is a timeout scheduled to release this
4474                  * sim queue, remove it.  The queue frozen count is
4475                  * already at 0.
4476                  */
4477                 if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4478                         callout_stop(&sim->callout);
4479                         sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4480                 }
4481                 if (run_queue) {
4482                         /*
4483                          * Now that we are unfrozen run the send queue.
4484                          */
4485                         xpt_run_devq(sim->devq);
4486                 }
4487         }
4488         mtx_unlock(&devq->send_mtx);
4489 }
4490
4491 /*
4492  * XXX Appears to be unused.
4493  */
4494 static void
4495 xpt_release_simq_timeout(void *arg)
4496 {
4497         struct cam_sim *sim;
4498
4499         sim = (struct cam_sim *)arg;
4500         xpt_release_simq(sim, /* run_queue */ TRUE);
4501 }
4502
4503 void
4504 xpt_done(union ccb *done_ccb)
4505 {
4506         struct cam_doneq *queue;
4507         int     run, hash;
4508
4509         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
4510             ("xpt_done: func= %#x %s status %#x\n",
4511                 done_ccb->ccb_h.func_code,
4512                 xpt_action_name(done_ccb->ccb_h.func_code),
4513                 done_ccb->ccb_h.status));
4514         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4515                 return;
4516
4517         /* Store the time the ccb was in the sim */
4518         done_ccb->ccb_h.qos.sim_data = sbinuptime() - done_ccb->ccb_h.qos.sim_data;
4519         hash = (done_ccb->ccb_h.path_id + done_ccb->ccb_h.target_id +
4520             done_ccb->ccb_h.target_lun) % cam_num_doneqs;
4521         queue = &cam_doneqs[hash];
4522         mtx_lock(&queue->cam_doneq_mtx);
4523         run = (queue->cam_doneq_sleep && STAILQ_EMPTY(&queue->cam_doneq));
4524         STAILQ_INSERT_TAIL(&queue->cam_doneq, &done_ccb->ccb_h, sim_links.stqe);
4525         done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4526         mtx_unlock(&queue->cam_doneq_mtx);
4527         if (run)
4528                 wakeup(&queue->cam_doneq);
4529 }
4530
4531 void
4532 xpt_done_direct(union ccb *done_ccb)
4533 {
4534
4535         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
4536             ("xpt_done_direct: status %#x\n", done_ccb->ccb_h.status));
4537         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0)
4538                 return;
4539
4540         /* Store the time the ccb was in the sim */
4541         done_ccb->ccb_h.qos.sim_data = sbinuptime() - done_ccb->ccb_h.qos.sim_data;
4542         xpt_done_process(&done_ccb->ccb_h);
4543 }
4544
4545 union ccb *
4546 xpt_alloc_ccb()
4547 {
4548         union ccb *new_ccb;
4549
4550         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4551         return (new_ccb);
4552 }
4553
4554 union ccb *
4555 xpt_alloc_ccb_nowait()
4556 {
4557         union ccb *new_ccb;
4558
4559         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4560         return (new_ccb);
4561 }
4562
4563 void
4564 xpt_free_ccb(union ccb *free_ccb)
4565 {
4566         free(free_ccb, M_CAMCCB);
4567 }
4568
4569
4570
4571 /* Private XPT functions */
4572
4573 /*
4574  * Get a CAM control block for the caller. Charge the structure to the device
4575  * referenced by the path.  If we don't have sufficient resources to allocate
4576  * more ccbs, we return NULL.
4577  */
4578 static union ccb *
4579 xpt_get_ccb_nowait(struct cam_periph *periph)
4580 {
4581         union ccb *new_ccb;
4582
4583         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_NOWAIT);
4584         if (new_ccb == NULL)
4585                 return (NULL);
4586         periph->periph_allocated++;
4587         cam_ccbq_take_opening(&periph->path->device->ccbq);
4588         return (new_ccb);
4589 }
4590
4591 static union ccb *
4592 xpt_get_ccb(struct cam_periph *periph)
4593 {
4594         union ccb *new_ccb;
4595
4596         cam_periph_unlock(periph);
4597         new_ccb = malloc(sizeof(*new_ccb), M_CAMCCB, M_ZERO|M_WAITOK);
4598         cam_periph_lock(periph);
4599         periph->periph_allocated++;
4600         cam_ccbq_take_opening(&periph->path->device->ccbq);
4601         return (new_ccb);
4602 }
4603
4604 union ccb *
4605 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
4606 {
4607         struct ccb_hdr *ccb_h;
4608
4609         CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("cam_periph_getccb\n"));
4610         cam_periph_assert(periph, MA_OWNED);
4611         while ((ccb_h = SLIST_FIRST(&periph->ccb_list)) == NULL ||
4612             ccb_h->pinfo.priority != priority) {
4613                 if (priority < periph->immediate_priority) {
4614                         periph->immediate_priority = priority;
4615                         xpt_run_allocq(periph, 0);
4616                 } else
4617                         cam_periph_sleep(periph, &periph->ccb_list, PRIBIO,
4618                             "cgticb", 0);
4619         }
4620         SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
4621         return ((union ccb *)ccb_h);
4622 }
4623
4624 static void
4625 xpt_acquire_bus(struct cam_eb *bus)
4626 {
4627
4628         xpt_lock_buses();
4629         bus->refcount++;
4630         xpt_unlock_buses();
4631 }
4632
4633 static void
4634 xpt_release_bus(struct cam_eb *bus)
4635 {
4636
4637         xpt_lock_buses();
4638         KASSERT(bus->refcount >= 1, ("bus->refcount >= 1"));
4639         if (--bus->refcount > 0) {
4640                 xpt_unlock_buses();
4641                 return;
4642         }
4643         TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
4644         xsoftc.bus_generation++;
4645         xpt_unlock_buses();
4646         KASSERT(TAILQ_EMPTY(&bus->et_entries),
4647             ("destroying bus, but target list is not empty"));
4648         cam_sim_release(bus->sim);
4649         mtx_destroy(&bus->eb_mtx);
4650         free(bus, M_CAMXPT);
4651 }
4652
4653 static struct cam_et *
4654 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4655 {
4656         struct cam_et *cur_target, *target;
4657
4658         mtx_assert(&xsoftc.xpt_topo_lock, MA_OWNED);
4659         mtx_assert(&bus->eb_mtx, MA_OWNED);
4660         target = (struct cam_et *)malloc(sizeof(*target), M_CAMXPT,
4661                                          M_NOWAIT|M_ZERO);
4662         if (target == NULL)
4663                 return (NULL);
4664
4665         TAILQ_INIT(&target->ed_entries);
4666         target->bus = bus;
4667         target->target_id = target_id;
4668         target->refcount = 1;
4669         target->generation = 0;
4670         target->luns = NULL;
4671         mtx_init(&target->luns_mtx, "CAM LUNs lock", NULL, MTX_DEF);
4672         timevalclear(&target->last_reset);
4673         /*
4674          * Hold a reference to our parent bus so it
4675          * will not go away before we do.
4676          */
4677         bus->refcount++;
4678
4679         /* Insertion sort into our bus's target list */
4680         cur_target = TAILQ_FIRST(&bus->et_entries);
4681         while (cur_target != NULL && cur_target->target_id < target_id)
4682                 cur_target = TAILQ_NEXT(cur_target, links);
4683         if (cur_target != NULL) {
4684                 TAILQ_INSERT_BEFORE(cur_target, target, links);
4685         } else {
4686                 TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4687         }
4688         bus->generation++;
4689         return (target);
4690 }
4691
4692 static void
4693 xpt_acquire_target(struct cam_et *target)
4694 {
4695         struct cam_eb *bus = target->bus;
4696
4697         mtx_lock(&bus->eb_mtx);
4698         target->refcount++;
4699         mtx_unlock(&bus->eb_mtx);
4700 }
4701
4702 static void
4703 xpt_release_target(struct cam_et *target)
4704 {
4705         struct cam_eb *bus = target->bus;
4706
4707         mtx_lock(&bus->eb_mtx);
4708         if (--target->refcount > 0) {
4709                 mtx_unlock(&bus->eb_mtx);
4710                 return;
4711         }
4712         TAILQ_REMOVE(&bus->et_entries, target, links);
4713         bus->generation++;
4714         mtx_unlock(&bus->eb_mtx);
4715         KASSERT(TAILQ_EMPTY(&target->ed_entries),
4716             ("destroying target, but device list is not empty"));
4717         xpt_release_bus(bus);
4718         mtx_destroy(&target->luns_mtx);
4719         if (target->luns)
4720                 free(target->luns, M_CAMXPT);
4721         free(target, M_CAMXPT);
4722 }
4723
4724 static struct cam_ed *
4725 xpt_alloc_device_default(struct cam_eb *bus, struct cam_et *target,
4726                          lun_id_t lun_id)
4727 {
4728         struct cam_ed *device;
4729
4730         device = xpt_alloc_device(bus, target, lun_id);
4731         if (device == NULL)
4732                 return (NULL);
4733
4734         device->mintags = 1;
4735         device->maxtags = 1;
4736         return (device);
4737 }
4738
4739 static void
4740 xpt_destroy_device(void *context, int pending)
4741 {
4742         struct cam_ed   *device = context;
4743
4744         mtx_lock(&device->device_mtx);
4745         mtx_destroy(&device->device_mtx);
4746         free(device, M_CAMDEV);
4747 }
4748
4749 struct cam_ed *
4750 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4751 {
4752         struct cam_ed   *cur_device, *device;
4753         struct cam_devq *devq;
4754         cam_status status;
4755
4756         mtx_assert(&bus->eb_mtx, MA_OWNED);
4757         /* Make space for us in the device queue on our bus */
4758         devq = bus->sim->devq;
4759         mtx_lock(&devq->send_mtx);
4760         status = cam_devq_resize(devq, devq->send_queue.array_size + 1);
4761         mtx_unlock(&devq->send_mtx);
4762         if (status != CAM_REQ_CMP)
4763                 return (NULL);
4764
4765         device = (struct cam_ed *)malloc(sizeof(*device),
4766                                          M_CAMDEV, M_NOWAIT|M_ZERO);
4767         if (device == NULL)
4768                 return (NULL);
4769
4770         cam_init_pinfo(&device->devq_entry);
4771         device->target = target;
4772         device->lun_id = lun_id;
4773         device->sim = bus->sim;
4774         if (cam_ccbq_init(&device->ccbq,
4775                           bus->sim->max_dev_openings) != 0) {
4776                 free(device, M_CAMDEV);
4777                 return (NULL);
4778         }
4779         SLIST_INIT(&device->asyncs);
4780         SLIST_INIT(&device->periphs);
4781         device->generation = 0;
4782         device->flags = CAM_DEV_UNCONFIGURED;
4783         device->tag_delay_count = 0;
4784         device->tag_saved_openings = 0;
4785         device->refcount = 1;
4786         mtx_init(&device->device_mtx, "CAM device lock", NULL, MTX_DEF);
4787         callout_init_mtx(&device->callout, &devq->send_mtx, 0);
4788         TASK_INIT(&device->device_destroy_task, 0, xpt_destroy_device, device);
4789         /*
4790          * Hold a reference to our parent bus so it
4791          * will not go away before we do.
4792          */
4793         target->refcount++;
4794
4795         cur_device = TAILQ_FIRST(&target->ed_entries);
4796         while (cur_device != NULL && cur_device->lun_id < lun_id)
4797                 cur_device = TAILQ_NEXT(cur_device, links);
4798         if (cur_device != NULL)
4799                 TAILQ_INSERT_BEFORE(cur_device, device, links);
4800         else
4801                 TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4802         target->generation++;
4803         return (device);
4804 }
4805
4806 void
4807 xpt_acquire_device(struct cam_ed *device)
4808 {
4809         struct cam_eb *bus = device->target->bus;
4810
4811         mtx_lock(&bus->eb_mtx);
4812         device->refcount++;
4813         mtx_unlock(&bus->eb_mtx);
4814 }
4815
4816 void
4817 xpt_release_device(struct cam_ed *device)
4818 {
4819         struct cam_eb *bus = device->target->bus;
4820         struct cam_devq *devq;
4821
4822         mtx_lock(&bus->eb_mtx);
4823         if (--device->refcount > 0) {
4824                 mtx_unlock(&bus->eb_mtx);
4825                 return;
4826         }
4827
4828         TAILQ_REMOVE(&device->target->ed_entries, device,links);
4829         device->target->generation++;
4830         mtx_unlock(&bus->eb_mtx);
4831
4832         /* Release our slot in the devq */
4833         devq = bus->sim->devq;
4834         mtx_lock(&devq->send_mtx);
4835         cam_devq_resize(devq, devq->send_queue.array_size - 1);
4836         mtx_unlock(&devq->send_mtx);
4837
4838         KASSERT(SLIST_EMPTY(&device->periphs),
4839             ("destroying device, but periphs list is not empty"));
4840         KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
4841             ("destroying device while still queued for ccbs"));
4842
4843         if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4844                 callout_stop(&device->callout);
4845
4846         xpt_release_target(device->target);
4847
4848         cam_ccbq_fini(&device->ccbq);
4849         /*
4850          * Free allocated memory.  free(9) does nothing if the
4851          * supplied pointer is NULL, so it is safe to call without
4852          * checking.
4853          */
4854         free(device->supported_vpds, M_CAMXPT);
4855         free(device->device_id, M_CAMXPT);
4856         free(device->ext_inq, M_CAMXPT);
4857         free(device->physpath, M_CAMXPT);
4858         free(device->rcap_buf, M_CAMXPT);
4859         free(device->serial_num, M_CAMXPT);
4860         taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
4861 }
4862
4863 u_int32_t
4864 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4865 {
4866         int     result;
4867         struct  cam_ed *dev;
4868
4869         dev = path->device;
4870         mtx_lock(&dev->sim->devq->send_mtx);
4871         result = cam_ccbq_resize(&dev->ccbq, newopenings);
4872         mtx_unlock(&dev->sim->devq->send_mtx);
4873         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
4874          || (dev->inq_flags & SID_CmdQue) != 0)
4875                 dev->tag_saved_openings = newopenings;
4876         return (result);
4877 }
4878
4879 static struct cam_eb *
4880 xpt_find_bus(path_id_t path_id)
4881 {
4882         struct cam_eb *bus;
4883
4884         xpt_lock_buses();
4885         for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
4886              bus != NULL;
4887              bus = TAILQ_NEXT(bus, links)) {
4888                 if (bus->path_id == path_id) {
4889                         bus->refcount++;
4890                         break;
4891                 }
4892         }
4893         xpt_unlock_buses();
4894         return (bus);
4895 }
4896
4897 static struct cam_et *
4898 xpt_find_target(struct cam_eb *bus, target_id_t target_id)
4899 {
4900         struct cam_et *target;
4901
4902         mtx_assert(&bus->eb_mtx, MA_OWNED);
4903         for (target = TAILQ_FIRST(&bus->et_entries);
4904              target != NULL;
4905              target = TAILQ_NEXT(target, links)) {
4906                 if (target->target_id == target_id) {
4907                         target->refcount++;
4908                         break;
4909                 }
4910         }
4911         return (target);
4912 }
4913
4914 static struct cam_ed *
4915 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4916 {
4917         struct cam_ed *device;
4918
4919         mtx_assert(&target->bus->eb_mtx, MA_OWNED);
4920         for (device = TAILQ_FIRST(&target->ed_entries);
4921              device != NULL;
4922              device = TAILQ_NEXT(device, links)) {
4923                 if (device->lun_id == lun_id) {
4924                         device->refcount++;
4925                         break;
4926                 }
4927         }
4928         return (device);
4929 }
4930
4931 void
4932 xpt_start_tags(struct cam_path *path)
4933 {
4934         struct ccb_relsim crs;
4935         struct cam_ed *device;
4936         struct cam_sim *sim;
4937         int    newopenings;
4938
4939         device = path->device;
4940         sim = path->bus->sim;
4941         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4942         xpt_freeze_devq(path, /*count*/1);
4943         device->inq_flags |= SID_CmdQue;
4944         if (device->tag_saved_openings != 0)
4945                 newopenings = device->tag_saved_openings;
4946         else
4947                 newopenings = min(device->maxtags,
4948                                   sim->max_tagged_dev_openings);
4949         xpt_dev_ccbq_resize(path, newopenings);
4950         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4951         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4952         crs.ccb_h.func_code = XPT_REL_SIMQ;
4953         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4954         crs.openings
4955             = crs.release_timeout
4956             = crs.qfrozen_cnt
4957             = 0;
4958         xpt_action((union ccb *)&crs);
4959 }
4960
4961 void
4962 xpt_stop_tags(struct cam_path *path)
4963 {
4964         struct ccb_relsim crs;
4965         struct cam_ed *device;
4966         struct cam_sim *sim;
4967
4968         device = path->device;
4969         sim = path->bus->sim;
4970         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
4971         device->tag_delay_count = 0;
4972         xpt_freeze_devq(path, /*count*/1);
4973         device->inq_flags &= ~SID_CmdQue;
4974         xpt_dev_ccbq_resize(path, sim->max_dev_openings);
4975         xpt_async(AC_GETDEV_CHANGED, path, NULL);
4976         xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL);
4977         crs.ccb_h.func_code = XPT_REL_SIMQ;
4978         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
4979         crs.openings
4980             = crs.release_timeout
4981             = crs.qfrozen_cnt
4982             = 0;
4983         xpt_action((union ccb *)&crs);
4984 }
4985
4986 static void
4987 xpt_boot_delay(void *arg)
4988 {
4989
4990         xpt_release_boot();
4991 }
4992
4993 static void
4994 xpt_config(void *arg)
4995 {
4996         /*
4997          * Now that interrupts are enabled, go find our devices
4998          */
4999         if (taskqueue_start_threads(&xsoftc.xpt_taskq, 1, PRIBIO, "CAM taskq"))
5000                 printf("xpt_config: failed to create taskqueue thread.\n");
5001
5002         /* Setup debugging path */
5003         if (cam_dflags != CAM_DEBUG_NONE) {
5004                 if (xpt_create_path(&cam_dpath, NULL,
5005                                     CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
5006                                     CAM_DEBUG_LUN) != CAM_REQ_CMP) {
5007                         printf("xpt_config: xpt_create_path() failed for debug"
5008                                " target %d:%d:%d, debugging disabled\n",
5009                                CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
5010                         cam_dflags = CAM_DEBUG_NONE;
5011                 }
5012         } else
5013                 cam_dpath = NULL;
5014
5015         periphdriver_init(1);
5016         xpt_hold_boot();
5017         callout_init(&xsoftc.boot_callout, 1);
5018         callout_reset_sbt(&xsoftc.boot_callout, SBT_1MS * xsoftc.boot_delay, 0,
5019             xpt_boot_delay, NULL, 0);
5020         /* Fire up rescan thread. */
5021         if (kproc_kthread_add(xpt_scanner_thread, NULL, &cam_proc, NULL, 0, 0,
5022             "cam", "scanner")) {
5023                 printf("xpt_config: failed to create rescan thread.\n");
5024         }
5025 }
5026
5027 void
5028 xpt_hold_boot(void)
5029 {
5030         xpt_lock_buses();
5031         xsoftc.buses_to_config++;
5032         xpt_unlock_buses();
5033 }
5034
5035 void
5036 xpt_release_boot(void)
5037 {
5038         xpt_lock_buses();
5039         xsoftc.buses_to_config--;
5040         if (xsoftc.buses_to_config == 0 && xsoftc.buses_config_done == 0) {
5041                 struct  xpt_task *task;
5042
5043                 xsoftc.buses_config_done = 1;
5044                 xpt_unlock_buses();
5045                 /* Call manually because we don't have any busses */
5046                 task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
5047                 if (task != NULL) {
5048                         TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
5049                         taskqueue_enqueue(taskqueue_thread, &task->task);
5050                 }
5051         } else
5052                 xpt_unlock_buses();
5053 }
5054
5055 /*
5056  * If the given device only has one peripheral attached to it, and if that
5057  * peripheral is the passthrough driver, announce it.  This insures that the
5058  * user sees some sort of announcement for every peripheral in their system.
5059  */
5060 static int
5061 xptpassannouncefunc(struct cam_ed *device, void *arg)
5062 {
5063         struct cam_periph *periph;
5064         int i;
5065
5066         for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
5067              periph = SLIST_NEXT(periph, periph_links), i++);
5068
5069         periph = SLIST_FIRST(&device->periphs);
5070         if ((i == 1)
5071          && (strncmp(periph->periph_name, "pass", 4) == 0))
5072                 xpt_announce_periph(periph, NULL);
5073
5074         return(1);
5075 }
5076
5077 static void
5078 xpt_finishconfig_task(void *context, int pending)
5079 {
5080
5081         periphdriver_init(2);
5082         /*
5083          * Check for devices with no "standard" peripheral driver
5084          * attached.  For any devices like that, announce the
5085          * passthrough driver so the user will see something.
5086          */
5087         if (!bootverbose)
5088                 xpt_for_all_devices(xptpassannouncefunc, NULL);
5089
5090         /* Release our hook so that the boot can continue. */
5091         config_intrhook_disestablish(xsoftc.xpt_config_hook);
5092         free(xsoftc.xpt_config_hook, M_CAMXPT);
5093         xsoftc.xpt_config_hook = NULL;
5094
5095         free(context, M_CAMXPT);
5096 }
5097
5098 cam_status
5099 xpt_register_async(int event, ac_callback_t *cbfunc, void *cbarg,
5100                    struct cam_path *path)
5101 {
5102         struct ccb_setasync csa;
5103         cam_status status;
5104         int xptpath = 0;
5105
5106         if (path == NULL) {
5107                 status = xpt_create_path(&path, /*periph*/NULL, CAM_XPT_PATH_ID,
5108                                          CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5109                 if (status != CAM_REQ_CMP)
5110                         return (status);
5111                 xpt_path_lock(path);
5112                 xptpath = 1;
5113         }
5114
5115         xpt_setup_ccb(&csa.ccb_h, path, CAM_PRIORITY_NORMAL);
5116         csa.ccb_h.func_code = XPT_SASYNC_CB;
5117         csa.event_enable = event;
5118         csa.callback = cbfunc;
5119         csa.callback_arg = cbarg;
5120         xpt_action((union ccb *)&csa);
5121         status = csa.ccb_h.status;
5122
5123         CAM_DEBUG(csa.ccb_h.path, CAM_DEBUG_TRACE,
5124             ("xpt_register_async: func %p\n", cbfunc));
5125
5126         if (xptpath) {
5127                 xpt_path_unlock(path);
5128                 xpt_free_path(path);
5129         }
5130
5131         if ((status == CAM_REQ_CMP) &&
5132             (csa.event_enable & AC_FOUND_DEVICE)) {
5133                 /*
5134                  * Get this peripheral up to date with all
5135                  * the currently existing devices.
5136                  */
5137                 xpt_for_all_devices(xptsetasyncfunc, &csa);
5138         }
5139         if ((status == CAM_REQ_CMP) &&
5140             (csa.event_enable & AC_PATH_REGISTERED)) {
5141                 /*
5142                  * Get this peripheral up to date with all
5143                  * the currently existing busses.
5144                  */
5145                 xpt_for_all_busses(xptsetasyncbusfunc, &csa);
5146         }
5147
5148         return (status);
5149 }
5150
5151 static void
5152 xptaction(struct cam_sim *sim, union ccb *work_ccb)
5153 {
5154         CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
5155
5156         switch (work_ccb->ccb_h.func_code) {
5157         /* Common cases first */
5158         case XPT_PATH_INQ:              /* Path routing inquiry */
5159         {
5160                 struct ccb_pathinq *cpi;
5161
5162                 cpi = &work_ccb->cpi;
5163                 cpi->version_num = 1; /* XXX??? */
5164                 cpi->hba_inquiry = 0;
5165                 cpi->target_sprt = 0;
5166                 cpi->hba_misc = 0;
5167                 cpi->hba_eng_cnt = 0;
5168                 cpi->max_target = 0;
5169                 cpi->max_lun = 0;
5170                 cpi->initiator_id = 0;
5171                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5172                 strlcpy(cpi->hba_vid, "", HBA_IDLEN);
5173                 strlcpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
5174                 cpi->unit_number = sim->unit_number;
5175                 cpi->bus_id = sim->bus_id;
5176                 cpi->base_transfer_speed = 0;
5177                 cpi->protocol = PROTO_UNSPECIFIED;
5178                 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
5179                 cpi->transport = XPORT_UNSPECIFIED;
5180                 cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
5181                 cpi->ccb_h.status = CAM_REQ_CMP;
5182                 xpt_done(work_ccb);
5183                 break;
5184         }
5185         default:
5186                 work_ccb->ccb_h.status = CAM_REQ_INVALID;
5187                 xpt_done(work_ccb);
5188                 break;
5189         }
5190 }
5191
5192 /*
5193  * The xpt as a "controller" has no interrupt sources, so polling
5194  * is a no-op.
5195  */
5196 static void
5197 xptpoll(struct cam_sim *sim)
5198 {
5199 }
5200
5201 void
5202 xpt_lock_buses(void)
5203 {
5204         mtx_lock(&xsoftc.xpt_topo_lock);
5205 }
5206
5207 void
5208 xpt_unlock_buses(void)
5209 {
5210         mtx_unlock(&xsoftc.xpt_topo_lock);
5211 }
5212
5213 struct mtx *
5214 xpt_path_mtx(struct cam_path *path)
5215 {
5216
5217         return (&path->device->device_mtx);
5218 }
5219
5220 static void
5221 xpt_done_process(struct ccb_hdr *ccb_h)
5222 {
5223         struct cam_sim *sim;
5224         struct cam_devq *devq;
5225         struct mtx *mtx = NULL;
5226
5227         if (ccb_h->flags & CAM_HIGH_POWER) {
5228                 struct highpowerlist    *hphead;
5229                 struct cam_ed           *device;
5230
5231                 mtx_lock(&xsoftc.xpt_highpower_lock);
5232                 hphead = &xsoftc.highpowerq;
5233
5234                 device = STAILQ_FIRST(hphead);
5235
5236                 /*
5237                  * Increment the count since this command is done.
5238                  */
5239                 xsoftc.num_highpower++;
5240
5241                 /*
5242                  * Any high powered commands queued up?
5243                  */
5244                 if (device != NULL) {
5245
5246                         STAILQ_REMOVE_HEAD(hphead, highpowerq_entry);
5247                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5248
5249                         mtx_lock(&device->sim->devq->send_mtx);
5250                         xpt_release_devq_device(device,
5251                                          /*count*/1, /*runqueue*/TRUE);
5252                         mtx_unlock(&device->sim->devq->send_mtx);
5253                 } else
5254                         mtx_unlock(&xsoftc.xpt_highpower_lock);
5255         }
5256
5257         sim = ccb_h->path->bus->sim;
5258
5259         if (ccb_h->status & CAM_RELEASE_SIMQ) {
5260                 xpt_release_simq(sim, /*run_queue*/FALSE);
5261                 ccb_h->status &= ~CAM_RELEASE_SIMQ;
5262         }
5263
5264         if ((ccb_h->flags & CAM_DEV_QFRZDIS)
5265          && (ccb_h->status & CAM_DEV_QFRZN)) {
5266                 xpt_release_devq(ccb_h->path, /*count*/1, /*run_queue*/TRUE);
5267                 ccb_h->status &= ~CAM_DEV_QFRZN;
5268         }
5269
5270         devq = sim->devq;
5271         if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
5272                 struct cam_ed *dev = ccb_h->path->device;
5273
5274                 mtx_lock(&devq->send_mtx);
5275                 devq->send_active--;
5276                 devq->send_openings++;
5277                 cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
5278
5279                 if (((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
5280                   && (dev->ccbq.dev_active == 0))) {
5281                         dev->flags &= ~CAM_DEV_REL_ON_QUEUE_EMPTY;
5282                         xpt_release_devq_device(dev, /*count*/1,
5283                                          /*run_queue*/FALSE);
5284                 }
5285
5286                 if (((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
5287                   && (ccb_h->status&CAM_STATUS_MASK) != CAM_REQUEUE_REQ)) {
5288                         dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
5289                         xpt_release_devq_device(dev, /*count*/1,
5290                                          /*run_queue*/FALSE);
5291                 }
5292
5293                 if (!device_is_queued(dev))
5294                         (void)xpt_schedule_devq(devq, dev);
5295                 xpt_run_devq(devq);
5296                 mtx_unlock(&devq->send_mtx);
5297
5298                 if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0) {
5299                         mtx = xpt_path_mtx(ccb_h->path);
5300                         mtx_lock(mtx);
5301
5302                         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5303                          && (--dev->tag_delay_count == 0))
5304                                 xpt_start_tags(ccb_h->path);
5305                 }
5306         }
5307
5308         if ((ccb_h->flags & CAM_UNLOCKED) == 0) {
5309                 if (mtx == NULL) {
5310                         mtx = xpt_path_mtx(ccb_h->path);
5311                         mtx_lock(mtx);
5312                 }
5313         } else {
5314                 if (mtx != NULL) {
5315                         mtx_unlock(mtx);
5316                         mtx = NULL;
5317                 }
5318         }
5319
5320         /* Call the peripheral driver's callback */
5321         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
5322         (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
5323         if (mtx != NULL)
5324                 mtx_unlock(mtx);
5325 }
5326
5327 void
5328 xpt_done_td(void *arg)
5329 {
5330         struct cam_doneq *queue = arg;
5331         struct ccb_hdr *ccb_h;
5332         STAILQ_HEAD(, ccb_hdr)  doneq;
5333
5334         STAILQ_INIT(&doneq);
5335         mtx_lock(&queue->cam_doneq_mtx);
5336         while (1) {
5337                 while (STAILQ_EMPTY(&queue->cam_doneq)) {
5338                         queue->cam_doneq_sleep = 1;
5339                         msleep(&queue->cam_doneq, &queue->cam_doneq_mtx,
5340                             PRIBIO, "-", 0);
5341                         queue->cam_doneq_sleep = 0;
5342                 }
5343                 STAILQ_CONCAT(&doneq, &queue->cam_doneq);
5344                 mtx_unlock(&queue->cam_doneq_mtx);
5345
5346                 THREAD_NO_SLEEPING();
5347                 while ((ccb_h = STAILQ_FIRST(&doneq)) != NULL) {
5348                         STAILQ_REMOVE_HEAD(&doneq, sim_links.stqe);
5349                         xpt_done_process(ccb_h);
5350                 }
5351                 THREAD_SLEEPING_OK();
5352
5353                 mtx_lock(&queue->cam_doneq_mtx);
5354         }
5355 }
5356
5357 static void
5358 camisr_runqueue(void)
5359 {
5360         struct  ccb_hdr *ccb_h;
5361         struct cam_doneq *queue;
5362         int i;
5363
5364         /* Process global queues. */
5365         for (i = 0; i < cam_num_doneqs; i++) {
5366                 queue = &cam_doneqs[i];
5367                 mtx_lock(&queue->cam_doneq_mtx);
5368                 while ((ccb_h = STAILQ_FIRST(&queue->cam_doneq)) != NULL) {
5369                         STAILQ_REMOVE_HEAD(&queue->cam_doneq, sim_links.stqe);
5370                         mtx_unlock(&queue->cam_doneq_mtx);
5371                         xpt_done_process(ccb_h);
5372                         mtx_lock(&queue->cam_doneq_mtx);
5373                 }
5374                 mtx_unlock(&queue->cam_doneq_mtx);
5375         }
5376 }
5377
5378 struct kv
5379 {
5380         uint32_t v;
5381         const char *name;
5382 };
5383
5384 static struct kv map[] = {
5385         { XPT_NOOP, "XPT_NOOP" },
5386         { XPT_SCSI_IO, "XPT_SCSI_IO" },
5387         { XPT_GDEV_TYPE, "XPT_GDEV_TYPE" },
5388         { XPT_GDEVLIST, "XPT_GDEVLIST" },
5389         { XPT_PATH_INQ, "XPT_PATH_INQ" },
5390         { XPT_REL_SIMQ, "XPT_REL_SIMQ" },
5391         { XPT_SASYNC_CB, "XPT_SASYNC_CB" },
5392         { XPT_SDEV_TYPE, "XPT_SDEV_TYPE" },
5393         { XPT_SCAN_BUS, "XPT_SCAN_BUS" },
5394         { XPT_DEV_MATCH, "XPT_DEV_MATCH" },
5395         { XPT_DEBUG, "XPT_DEBUG" },
5396         { XPT_PATH_STATS, "XPT_PATH_STATS" },
5397         { XPT_GDEV_STATS, "XPT_GDEV_STATS" },
5398         { XPT_DEV_ADVINFO, "XPT_DEV_ADVINFO" },
5399         { XPT_ASYNC, "XPT_ASYNC" },
5400         { XPT_ABORT, "XPT_ABORT" },
5401         { XPT_RESET_BUS, "XPT_RESET_BUS" },
5402         { XPT_RESET_DEV, "XPT_RESET_DEV" },
5403         { XPT_TERM_IO, "XPT_TERM_IO" },
5404         { XPT_SCAN_LUN, "XPT_SCAN_LUN" },
5405         { XPT_GET_TRAN_SETTINGS, "XPT_GET_TRAN_SETTINGS" },
5406         { XPT_SET_TRAN_SETTINGS, "XPT_SET_TRAN_SETTINGS" },
5407         { XPT_CALC_GEOMETRY, "XPT_CALC_GEOMETRY" },
5408         { XPT_ATA_IO, "XPT_ATA_IO" },
5409         { XPT_GET_SIM_KNOB, "XPT_GET_SIM_KNOB" },
5410         { XPT_SET_SIM_KNOB, "XPT_SET_SIM_KNOB" },
5411         { XPT_NVME_IO, "XPT_NVME_IO" },
5412         { XPT_MMCSD_IO, "XPT_MMCSD_IO" },
5413         { XPT_SMP_IO, "XPT_SMP_IO" },
5414         { XPT_SCAN_TGT, "XPT_SCAN_TGT" },
5415         { XPT_ENG_INQ, "XPT_ENG_INQ" },
5416         { XPT_ENG_EXEC, "XPT_ENG_EXEC" },
5417         { XPT_EN_LUN, "XPT_EN_LUN" },
5418         { XPT_TARGET_IO, "XPT_TARGET_IO" },
5419         { XPT_ACCEPT_TARGET_IO, "XPT_ACCEPT_TARGET_IO" },
5420         { XPT_CONT_TARGET_IO, "XPT_CONT_TARGET_IO" },
5421         { XPT_IMMED_NOTIFY, "XPT_IMMED_NOTIFY" },
5422         { XPT_NOTIFY_ACK, "XPT_NOTIFY_ACK" },
5423         { XPT_IMMEDIATE_NOTIFY, "XPT_IMMEDIATE_NOTIFY" },
5424         { XPT_NOTIFY_ACKNOWLEDGE, "XPT_NOTIFY_ACKNOWLEDGE" },
5425         { 0, 0 }
5426 };
5427
5428 static const char *
5429 xpt_action_name(uint32_t action)
5430 {
5431         static char buffer[32]; /* Only for unknown messages -- racy */
5432         struct kv *walker = map;
5433
5434         while (walker->name != NULL) {
5435                 if (walker->v == action)
5436                         return (walker->name);
5437                 walker++;
5438         }
5439
5440         snprintf(buffer, sizeof(buffer), "%#x", action);
5441         return (buffer);
5442 }