sys/cam/cam_xpt.c

   1 /*
   2  * Implementation of the Common Access Method Transport (XPT) layer.
   3  *
   4  * Copyright (c) 1997, 1998, 1999 Justin T. Gibbs.
   5  * Copyright (c) 1997, 1998, 1999 Kenneth D. Merry.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification, immediately at the beginning of the file.
  14  * 2. The name of the author may not be used to endorse or promote products
  15  *    derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/types.h>
  34 #include <sys/malloc.h>
  35 #include <sys/kernel.h>
  36 #include <sys/time.h>
  37 #include <sys/conf.h>
  38 #include <sys/fcntl.h>
  39 #include <sys/md5.h>
  40 #include <sys/devicestat.h>
  41 #include <sys/interrupt.h>
  42 #include <sys/bus.h>
  43
  44 #ifdef PC98
  45 #include <pc98/pc98/pc98_machdep.h>     /* geometry translation */
  46 #endif
  47
  48 #include <machine/clock.h>
  49 #include <machine/ipl.h>
  50
  51 #include <cam/cam.h>
  52 #include <cam/cam_ccb.h>
  53 #include <cam/cam_periph.h>
  54 #include <cam/cam_sim.h>
  55 #include <cam/cam_xpt.h>
  56 #include <cam/cam_xpt_sim.h>
  57 #include <cam/cam_xpt_periph.h>
  58 #include <cam/cam_debug.h>
  59
  60 #include <cam/scsi/scsi_all.h>
  61 #include <cam/scsi/scsi_message.h>
  62 #include <cam/scsi/scsi_pass.h>
  63 #include "opt_cam.h"
  64
  65 /* Datastructures internal to the xpt layer */
  66
  67 /*
  68  * Definition of an async handler callback block.  These are used to add
  69  * SIMs and peripherals to the async callback lists.
  70  */
  71 struct async_node {
  72         SLIST_ENTRY(async_node) links;
  73         u_int32_t       event_enable;   /* Async Event enables */
  74         void            (*callback)(void *arg, u_int32_t code,
  75                                     struct cam_path *path, void *args);
  76         void            *callback_arg;
  77 };
  78
  79 SLIST_HEAD(async_list, async_node);
  80 SLIST_HEAD(periph_list, cam_periph);
  81 static STAILQ_HEAD(highpowerlist, ccb_hdr) highpowerq;
  82
  83 /*
  84  * This is the maximum number of high powered commands (e.g. start unit)
  85  * that can be outstanding at a particular time.
  86  */
  87 #ifndef CAM_MAX_HIGHPOWER
  88 #define CAM_MAX_HIGHPOWER  4
  89 #endif
  90
  91 /* number of high powered commands that can go through right now */
  92 static int num_highpower = CAM_MAX_HIGHPOWER;
  93
  94 /*
  95  * Structure for queueing a device in a run queue.
  96  * There is one run queue for allocating new ccbs,
  97  * and another for sending ccbs to the controller.
  98  */
  99 struct cam_ed_qinfo {
 100         cam_pinfo pinfo;
 101         struct    cam_ed *device;
 102 };
 103
 104 /*
 105  * The CAM EDT (Existing Device Table) contains the device information for
 106  * all devices for all busses in the system.  The table contains a
 107  * cam_ed structure for each device on the bus.
 108  */
 109 struct cam_ed {
 110         TAILQ_ENTRY(cam_ed) links;
 111         struct  cam_ed_qinfo alloc_ccb_entry;
 112         struct  cam_ed_qinfo send_ccb_entry;
 113         struct  cam_et   *target;
 114         lun_id_t         lun_id;
 115         struct  camq drvq;              /*
 116                                          * Queue of type drivers wanting to do
 117                                          * work on this device.
 118                                          */
 119         struct  cam_ccbq ccbq;          /* Queue of pending ccbs */
 120         struct  async_list asyncs;      /* Async callback info for this B/T/L */
 121         struct  periph_list periphs;    /* All attached devices */
 122         u_int   generation;             /* Generation number */
 123         struct  cam_periph *owner;      /* Peripheral driver's ownership tag */
 124         struct  xpt_quirk_entry *quirk; /* Oddities about this device */
 125                                         /* Storage for the inquiry data */
 126         struct  scsi_inquiry_data inq_data;
 127         u_int8_t         inq_flags;     /*
 128                                          * Current settings for inquiry flags.
 129                                          * This allows us to override settings
 130                                          * like disconnection and tagged
 131                                          * queuing for a device.
 132                                          */
 133         u_int8_t         queue_flags;   /* Queue flags from the control page */
 134         u_int8_t         serial_num_len;
 135         u_int8_t         *serial_num;
 136         u_int32_t        qfrozen_cnt;
 137         u_int32_t        flags;
 138 #define CAM_DEV_UNCONFIGURED            0x01
 139 #define CAM_DEV_REL_TIMEOUT_PENDING     0x02
 140 #define CAM_DEV_REL_ON_COMPLETE         0x04
 141 #define CAM_DEV_REL_ON_QUEUE_EMPTY      0x08
 142 #define CAM_DEV_RESIZE_QUEUE_NEEDED     0x10
 143 #define CAM_DEV_TAG_AFTER_COUNT         0x20
 144 #define CAM_DEV_INQUIRY_DATA_VALID      0x40
 145         u_int32_t        tag_delay_count;
 146 #define CAM_TAG_DELAY_COUNT             5
 147         u_int32_t        refcount;
 148         struct           callout_handle c_handle;
 149 };
 150
 151 /*
 152  * Each target is represented by an ET (Existing Target).  These
 153  * entries are created when a target is successfully probed with an
 154  * identify, and removed when a device fails to respond after a number
 155  * of retries, or a bus rescan finds the device missing.
 156  */
 157 struct cam_et {
 158         TAILQ_HEAD(, cam_ed) ed_entries;
 159         TAILQ_ENTRY(cam_et) links;
 160         struct  cam_eb  *bus;
 161         target_id_t     target_id;
 162         u_int32_t       refcount;
 163         u_int           generation;
 164         struct          timeval last_reset;
 165 };
 166
 167 /*
 168  * Each bus is represented by an EB (Existing Bus).  These entries
 169  * are created by calls to xpt_bus_register and deleted by calls to
 170  * xpt_bus_deregister.
 171  */
 172 struct cam_eb {
 173         TAILQ_HEAD(, cam_et) et_entries;
 174         TAILQ_ENTRY(cam_eb)  links;
 175         path_id_t            path_id;
 176         struct cam_sim       *sim;
 177         struct timeval       last_reset;
 178         u_int32_t            flags;
 179 #define CAM_EB_RUNQ_SCHEDULED   0x01
 180         u_int32_t            refcount;
 181         u_int                generation;
 182 };
 183
 184 struct cam_path {
 185         struct cam_periph *periph;
 186         struct cam_eb     *bus;
 187         struct cam_et     *target;
 188         struct cam_ed     *device;
 189 };
 190
 191 struct xpt_quirk_entry {
 192         struct scsi_inquiry_pattern inq_pat;
 193         u_int8_t quirks;
 194 #define CAM_QUIRK_NOLUNS        0x01
 195 #define CAM_QUIRK_NOSERIAL      0x02
 196 #define CAM_QUIRK_HILUNS        0x04
 197         u_int mintags;
 198         u_int maxtags;
 199 };
 200 #define CAM_SCSI2_MAXLUN        8
 201
 202 typedef enum {
 203         XPT_FLAG_OPEN           = 0x01
 204 } xpt_flags;
 205
 206 struct xpt_softc {
 207         xpt_flags       flags;
 208         u_int32_t       generation;
 209 };
 210
 211 static const char quantum[] = "QUANTUM";
 212 static const char sony[] = "SONY";
 213 static const char west_digital[] = "WDIGTL";
 214 static const char samsung[] = "SAMSUNG";
 215 static const char seagate[] = "SEAGATE";
 216 static const char microp[] = "MICROP";
 217
 218 static struct xpt_quirk_entry xpt_quirk_table[] =
 219 {
 220         {
 221                 /* Reports QUEUE FULL for temporary resource shortages */
 222                 { T_DIRECT, SIP_MEDIA_FIXED, quantum, "XP39100*", "*" },
 223                 /*quirks*/0, /*mintags*/24, /*maxtags*/32
 224         },
 225         {
 226                 /* Reports QUEUE FULL for temporary resource shortages */
 227                 { T_DIRECT, SIP_MEDIA_FIXED, quantum, "XP34550*", "*" },
 228                 /*quirks*/0, /*mintags*/24, /*maxtags*/32
 229         },
 230         {
 231                 /* Reports QUEUE FULL for temporary resource shortages */
 232                 { T_DIRECT, SIP_MEDIA_FIXED, quantum, "XP32275*", "*" },
 233                 /*quirks*/0, /*mintags*/24, /*maxtags*/32
 234         },
 235         {
 236                 /* Broken tagged queuing drive */
 237                 { T_DIRECT, SIP_MEDIA_FIXED, microp, "4421-07*", "*" },
 238                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 239         },
 240         {
 241                 /* Broken tagged queuing drive */
 242                 { T_DIRECT, SIP_MEDIA_FIXED, "HP", "C372*", "*" },
 243                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 244         },
 245         {
 246                 /* Broken tagged queuing drive */
 247                 { T_DIRECT, SIP_MEDIA_FIXED, microp, "3391*", "x43h" },
 248                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 249         },
 250         {
 251                 /*
 252                  * Unfortunately, the Quantum Atlas III has the same
 253                  * problem as the Atlas II drives above.
 254                  * Reported by: "Johan Granlund" <johan@granlund.nu>
 255                  *
 256                  * For future reference, the drive with the problem was:
 257                  * QUANTUM QM39100TD-SW N1B0
 258                  *
 259                  * It's possible that Quantum will fix the problem in later
 260                  * firmware revisions.  If that happens, the quirk entry
 261                  * will need to be made specific to the firmware revisions
 262                  * with the problem.
 263                  *
 264                  */
 265                 /* Reports QUEUE FULL for temporary resource shortages */
 266                 { T_DIRECT, SIP_MEDIA_FIXED, quantum, "QM39100*", "*" },
 267                 /*quirks*/0, /*mintags*/24, /*maxtags*/32
 268         },
 269         {
 270                 /*
 271                  * 18 Gig Atlas III, same problem as the 9G version.
 272                  * Reported by: Andre Albsmeier
 273                  *              <andre.albsmeier@mchp.siemens.de>
 274                  *
 275                  * For future reference, the drive with the problem was:
 276                  * QUANTUM QM318000TD-S N491
 277                  */
 278                 /* Reports QUEUE FULL for temporary resource shortages */
 279                 { T_DIRECT, SIP_MEDIA_FIXED, quantum, "QM318000*", "*" },
 280                 /*quirks*/0, /*mintags*/24, /*maxtags*/32
 281         },
 282         {
 283                 /*
 284                  * Broken tagged queuing drive
 285                  * Reported by: Bret Ford <bford@uop.cs.uop.edu>
 286                  *         and: Martin Renters <martin@tdc.on.ca>
 287                  */
 288                 { T_DIRECT, SIP_MEDIA_FIXED, seagate, "ST410800*", "71*" },
 289                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 290         },
 291                 /*
 292                  * The Seagate Medalist Pro drives have very poor write
 293                  * performance with anything more than 2 tags.
 294                  *
 295                  * Reported by:  Paul van der Zwan <paulz@trantor.xs4all.nl>
 296                  * Drive:  <SEAGATE ST36530N 1444>
 297                  *
 298                  * Reported by:  Jeremy Lea <reg@shale.csir.co.za>
 299                  * Drive:  <SEAGATE ST34520W 1281>
 300                  *
 301                  * No one has actually reported that the 9G version
 302                  * (ST39140*) of the Medalist Pro has the same problem, but
 303                  * we're assuming that it does because the 4G and 6.5G
 304                  * versions of the drive are broken.
 305                  */
 306         {
 307                 { T_DIRECT, SIP_MEDIA_FIXED, seagate, "ST34520*", "*"},
 308                 /*quirks*/0, /*mintags*/2, /*maxtags*/2
 309         },
 310         {
 311                 { T_DIRECT, SIP_MEDIA_FIXED, seagate, "ST36530*", "*"},
 312                 /*quirks*/0, /*mintags*/2, /*maxtags*/2
 313         },
 314         {
 315                 { T_DIRECT, SIP_MEDIA_FIXED, seagate, "ST39140*", "*"},
 316                 /*quirks*/0, /*mintags*/2, /*maxtags*/2
 317         },
 318         {
 319                 /*
 320                  * Slow when tagged queueing is enabled.  Write performance
 321                  * steadily drops off with more and more concurrent
 322                  * transactions.  Best sequential write performance with
 323                  * tagged queueing turned off and write caching turned on.
 324                  *
 325                  * PR:  kern/10398
 326                  * Submitted by:  Hideaki Okada <hokada@isl.melco.co.jp>
 327                  * Drive:  DCAS-34330 w/ "S65A" firmware.
 328                  *
 329                  * The drive with the problem had the "S65A" firmware
 330                  * revision, and has also been reported (by Stephen J.
 331                  * Roznowski <sjr@home.net>) for a drive with the "S61A"
 332                  * firmware revision.
 333                  *
 334                  * Although no one has reported problems with the 2 gig
 335                  * version of the DCAS drive, the assumption is that it
 336                  * has the same problems as the 4 gig version.  Therefore
 337                  * this quirk entries disables tagged queueing for all
 338                  * DCAS drives.
 339                  */
 340                 { T_DIRECT, SIP_MEDIA_FIXED, "IBM", "DCAS*", "*" },
 341                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 342         },
 343         {
 344                 /* Broken tagged queuing drive */
 345                 { T_DIRECT, SIP_MEDIA_REMOVABLE, "iomega", "jaz*", "*" },
 346                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 347         },
 348         {
 349                 /* Broken tagged queuing drive */
 350                 { T_DIRECT, SIP_MEDIA_FIXED, "CONNER", "CFP2107*", "*" },
 351                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 352         },
 353         {
 354                 /*
 355                  * Broken tagged queuing drive.
 356                  * Submitted by:
 357                  * NAKAJI Hiroyuki <nakaji@zeisei.dpri.kyoto-u.ac.jp>
 358                  * in PR kern/9535
 359                  */
 360                 { T_DIRECT, SIP_MEDIA_FIXED, samsung, "WN34324U*", "*" },
 361                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 362         },
 363         {
 364                 /*
 365                  * Slow when tagged queueing is enabled. (1.5MB/sec versus
 366                  * 8MB/sec.)
 367                  * Submitted by: Andrew Gallatin <gallatin@cs.duke.edu>
 368                  * Best performance with these drives is achieved with
 369                  * tagged queueing turned off, and write caching turned on.
 370                  */
 371                 { T_DIRECT, SIP_MEDIA_FIXED, west_digital, "WDE*", "*" },
 372                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 373         },
 374         {
 375                 /*
 376                  * Slow when tagged queueing is enabled. (1.5MB/sec versus
 377                  * 8MB/sec.)
 378                  * Submitted by: Andrew Gallatin <gallatin@cs.duke.edu>
 379                  * Best performance with these drives is achieved with
 380                  * tagged queueing turned off, and write caching turned on.
 381                  */
 382                 { T_DIRECT, SIP_MEDIA_FIXED, west_digital, "ENTERPRISE", "*" },
 383                 /*quirks*/0, /*mintags*/0, /*maxtags*/0
 384         },
 385         {
 386                 /*
 387                  * Doesn't handle queue full condition correctly,
 388                  * so we need to limit maxtags to what the device
 389                  * can handle instead of determining this automatically.
 390                  */
 391                 { T_DIRECT, SIP_MEDIA_FIXED, samsung, "WN321010S*", "*" },
 392                 /*quirks*/0, /*mintags*/2, /*maxtags*/32
 393         },
 394         {
 395                 /* Really only one LUN */
 396                 { T_ENCLOSURE, SIP_MEDIA_FIXED, "SUN", "SENA*", "*" },
 397                 CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 398         },
 399         {
 400                 /* I can't believe we need a quirk for DPT volumes. */
 401                 { T_ANY, SIP_MEDIA_FIXED|SIP_MEDIA_REMOVABLE, "DPT", "*", "*" },
 402                 CAM_QUIRK_NOSERIAL|CAM_QUIRK_NOLUNS,
 403                 /*mintags*/0, /*maxtags*/255
 404         },
 405         {
 406                 /*
 407                  * Many Sony CDROM drives don't like multi-LUN probing.
 408                  */
 409                 { T_CDROM, SIP_MEDIA_REMOVABLE, sony, "CD-ROM CDU*", "*" },
 410                 CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 411         },
 412         {
 413                 /*
 414                  * This drive doesn't like multiple LUN probing.
 415                  * Submitted by:  Parag Patel <parag@cgt.com>
 416                  */
 417                 { T_WORM, SIP_MEDIA_REMOVABLE, sony, "CD-R   CDU9*", "*" },
 418                 CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 419         },
 420         {
 421                 /*
 422                  * The 8200 doesn't like multi-lun probing, and probably
 423                  * don't like serial number requests either.
 424                  */
 425                 {
 426                         T_SEQUENTIAL, SIP_MEDIA_REMOVABLE, "EXABYTE",
 427                         "EXB-8200*", "*"
 428                 },
 429                 CAM_QUIRK_NOSERIAL|CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 430         },
 431         {
 432                 /*
 433                  * These Hitachi drives don't like multi-lun probing.
 434                  * The PR submitter has a DK319H, but says that the Linux
 435                  * kernel has a similar work-around for the DK312 and DK314,
 436                  * so all DK31* drives are quirked here.
 437                  * PR:            misc/18793
 438                  * Submitted by:  Paul Haddad <paul@pth.com>
 439                  */
 440                 { T_DIRECT, SIP_MEDIA_FIXED, "HITACHI", "DK31*", "*" },
 441                 CAM_QUIRK_NOLUNS, /*mintags*/2, /*maxtags*/255
 442         },
 443         {
 444                 /*
 445                  * This old revision of the TDC3600 is also SCSI-1, and
 446                  * hangs upon serial number probing.
 447                  */
 448                 {
 449                         T_SEQUENTIAL, SIP_MEDIA_REMOVABLE, "TANDBERG",
 450                         " TDC 3600", "U07:"
 451                 },
 452                 CAM_QUIRK_NOSERIAL, /*mintags*/0, /*maxtags*/0
 453         },
 454         {
 455                 /*
 456                  * Would repond to all LUNs if asked for.
 457                  */
 458                 {
 459                         T_SEQUENTIAL, SIP_MEDIA_REMOVABLE, "CALIPER",
 460                         "CP150", "*"
 461                 },
 462                 CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 463         },
 464         {
 465                 /*
 466                  * Would repond to all LUNs if asked for.
 467                  */
 468                 {
 469                         T_SEQUENTIAL, SIP_MEDIA_REMOVABLE, "KENNEDY",
 470                         "96X2*", "*"
 471                 },
 472                 CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 473         },
 474         {
 475                 /* Submitted by: Matthew Dodd <winter@jurai.net> */
 476                 { T_PROCESSOR, SIP_MEDIA_FIXED, "Cabletrn", "EA41*", "*" },
 477                 CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 478         },
 479         {
 480                 /* Submitted by: Matthew Dodd <winter@jurai.net> */
 481                 { T_PROCESSOR, SIP_MEDIA_FIXED, "CABLETRN", "EA41*", "*" },
 482                 CAM_QUIRK_NOLUNS, /*mintags*/0, /*maxtags*/0
 483         },
 484         {
 485                 /* Default tagged queuing parameters for all devices */
 486                 {
 487                   T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
 488                   /*vendor*/"*", /*product*/"*", /*revision*/"*"
 489                 },
 490                 /*quirks*/0, /*mintags*/2, /*maxtags*/255
 491         },
 492 };
 493
 494 static const int xpt_quirk_table_size =
 495         sizeof(xpt_quirk_table) / sizeof(*xpt_quirk_table);
 496
 497 typedef enum {
 498         DM_RET_COPY             = 0x01,
 499         DM_RET_FLAG_MASK        = 0x0f,
 500         DM_RET_NONE             = 0x00,
 501         DM_RET_STOP             = 0x10,
 502         DM_RET_DESCEND          = 0x20,
 503         DM_RET_ERROR            = 0x30,
 504         DM_RET_ACTION_MASK      = 0xf0
 505 } dev_match_ret;
 506
 507 typedef enum {
 508         XPT_DEPTH_BUS,
 509         XPT_DEPTH_TARGET,
 510         XPT_DEPTH_DEVICE,
 511         XPT_DEPTH_PERIPH
 512 } xpt_traverse_depth;
 513
 514 struct xpt_traverse_config {
 515         xpt_traverse_depth      depth;
 516         void                    *tr_func;
 517         void                    *tr_arg;
 518 };
 519
 520 typedef int     xpt_busfunc_t (struct cam_eb *bus, void *arg);
 521 typedef int     xpt_targetfunc_t (struct cam_et *target, void *arg);
 522 typedef int     xpt_devicefunc_t (struct cam_ed *device, void *arg);
 523 typedef int     xpt_periphfunc_t (struct cam_periph *periph, void *arg);
 524 typedef int     xpt_pdrvfunc_t (struct periph_driver **pdrv, void *arg);
 525
 526 /* Transport layer configuration information */
 527 static struct xpt_softc xsoftc;
 528
 529 /* Queues for our software interrupt handler */
 530 typedef TAILQ_HEAD(cam_isrq, ccb_hdr) cam_isrq_t;
 531 static cam_isrq_t cam_bioq;
 532 static cam_isrq_t cam_netq;
 533
 534 /* "Pool" of inactive ccbs managed by xpt_alloc_ccb and xpt_free_ccb */
 535 static SLIST_HEAD(,ccb_hdr) ccb_freeq;
 536 static u_int xpt_max_ccbs;      /*
 537                                  * Maximum size of ccb pool.  Modified as
 538                                  * devices are added/removed or have their
 539                                  * opening counts changed.
 540                                  */
 541 static u_int xpt_ccb_count;     /* Current count of allocated ccbs */
 542
 543 struct cam_periph *xpt_periph;
 544
 545 static periph_init_t xpt_periph_init;
 546
 547 static periph_init_t probe_periph_init;
 548
 549 static struct periph_driver xpt_driver =
 550 {
 551         xpt_periph_init, "xpt",
 552         TAILQ_HEAD_INITIALIZER(xpt_driver.units)
 553 };
 554
 555 static struct periph_driver probe_driver =
 556 {
 557         probe_periph_init, "probe",
 558         TAILQ_HEAD_INITIALIZER(probe_driver.units)
 559 };
 560
 561 DATA_SET(periphdriver_set, xpt_driver);
 562 DATA_SET(periphdriver_set, probe_driver);
 563
 564 #define XPT_CDEV_MAJOR 104
 565
 566 static d_open_t xptopen;
 567 static d_close_t xptclose;
 568 static d_ioctl_t xptioctl;
 569
 570 static struct cdevsw xpt_cdevsw = {
 571         /* open */      xptopen,
 572         /* close */     xptclose,
 573         /* read */      noread,
 574         /* write */     nowrite,
 575         /* ioctl */     xptioctl,
 576         /* poll */      nopoll,
 577         /* mmap */      nommap,
 578         /* strategy */  nostrategy,
 579         /* name */      "xpt",
 580         /* maj */       XPT_CDEV_MAJOR,
 581         /* dump */      nodump,
 582         /* psize */     nopsize,
 583         /* flags */     0,
 584         /* bmaj */      -1
 585 };
 586
 587 static struct intr_config_hook *xpt_config_hook;
 588
 589 /* Registered busses */
 590 static TAILQ_HEAD(,cam_eb) xpt_busses;
 591 static u_int bus_generation;
 592
 593 /* Storage for debugging datastructures */
 594 #ifdef  CAMDEBUG
 595 struct cam_path *cam_dpath;
 596 u_int32_t cam_dflags;
 597 u_int32_t cam_debug_delay;
 598 #endif
 599
 600 #if defined(CAM_DEBUG_FLAGS) && !defined(CAMDEBUG)
 601 #error "You must have options CAMDEBUG to use options CAM_DEBUG_FLAGS"
 602 #endif
 603
 604 /*
 605  * In order to enable the CAM_DEBUG_* options, the user must have CAMDEBUG
 606  * enabled.  Also, the user must have either none, or all of CAM_DEBUG_BUS,
 607  * CAM_DEBUG_TARGET, and CAM_DEBUG_LUN specified.
 608  */
 609 #if defined(CAM_DEBUG_BUS) || defined(CAM_DEBUG_TARGET) \
 610     || defined(CAM_DEBUG_LUN)
 611 #ifdef CAMDEBUG
 612 #if !defined(CAM_DEBUG_BUS) || !defined(CAM_DEBUG_TARGET) \
 613     || !defined(CAM_DEBUG_LUN)
 614 #error "You must define all or none of CAM_DEBUG_BUS, CAM_DEBUG_TARGET \
 615         and CAM_DEBUG_LUN"
 616 #endif /* !CAM_DEBUG_BUS || !CAM_DEBUG_TARGET || !CAM_DEBUG_LUN */
 617 #else /* !CAMDEBUG */
 618 #error "You must use options CAMDEBUG if you use the CAM_DEBUG_* options"
 619 #endif /* CAMDEBUG */
 620 #endif /* CAM_DEBUG_BUS || CAM_DEBUG_TARGET || CAM_DEBUG_LUN */
 621
 622 /* Our boot-time initialization hook */
 623 static void     xpt_init(void *);
 624 SYSINIT(cam, SI_SUB_CONFIGURE, SI_ORDER_SECOND, xpt_init, NULL);
 625
 626 static cam_status       xpt_compile_path(struct cam_path *new_path,
 627                                          struct cam_periph *perph,
 628                                          path_id_t path_id,
 629                                          target_id_t target_id,
 630                                          lun_id_t lun_id);
 631
 632 static void             xpt_release_path(struct cam_path *path);
 633
 634 static void             xpt_async_bcast(struct async_list *async_head,
 635                                         u_int32_t async_code,
 636                                         struct cam_path *path,
 637                                         void *async_arg);
 638 static path_id_t xptnextfreepathid(void);
 639 static path_id_t xptpathid(const char *sim_name, int sim_unit, int sim_bus);
 640 static union ccb *xpt_get_ccb(struct cam_ed *device);
 641 static int       xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 642                                   u_int32_t new_priority);
 643 static void      xpt_run_dev_allocq(struct cam_eb *bus);
 644 static void      xpt_run_dev_sendq(struct cam_eb *bus);
 645 static timeout_t xpt_release_devq_timeout;
 646 static timeout_t xpt_release_simq_timeout;
 647 static void      xpt_release_bus(struct cam_eb *bus);
 648 static void      xpt_release_devq_device(struct cam_ed *dev, u_int count,
 649                                          int run_queue);
 650 static struct cam_et*
 651                  xpt_alloc_target(struct cam_eb *bus, target_id_t target_id);
 652 static void      xpt_release_target(struct cam_eb *bus, struct cam_et *target);
 653 static struct cam_ed*
 654                  xpt_alloc_device(struct cam_eb *bus, struct cam_et *target,
 655                                   lun_id_t lun_id);
 656 static void      xpt_release_device(struct cam_eb *bus, struct cam_et *target,
 657                                     struct cam_ed *device);
 658 static u_int32_t xpt_dev_ccbq_resize(struct cam_path *path, int newopenings);
 659 static struct cam_eb*
 660                  xpt_find_bus(path_id_t path_id);
 661 static struct cam_et*
 662                  xpt_find_target(struct cam_eb *bus, target_id_t target_id);
 663 static struct cam_ed*
 664                  xpt_find_device(struct cam_et *target, lun_id_t lun_id);
 665 static void      xpt_scan_bus(struct cam_periph *periph, union ccb *ccb);
 666 static void      xpt_scan_lun(struct cam_periph *periph,
 667                               struct cam_path *path, cam_flags flags,
 668                               union ccb *ccb);
 669 static void      xptscandone(struct cam_periph *periph, union ccb *done_ccb);
 670 static xpt_busfunc_t    xptconfigbuscountfunc;
 671 static xpt_busfunc_t    xptconfigfunc;
 672 static void      xpt_config(void *arg);
 673 static xpt_devicefunc_t xptpassannouncefunc;
 674 static void      xpt_finishconfig(struct cam_periph *periph, union ccb *ccb);
 675 static void      xptaction(struct cam_sim *sim, union ccb *work_ccb);
 676 static void      xptpoll(struct cam_sim *sim);
 677 static swihand_t swi_camnet;
 678 static swihand_t swi_cambio;
 679 static void      camisr(cam_isrq_t *queue);
 680 #if 0
 681 static void      xptstart(struct cam_periph *periph, union ccb *work_ccb);
 682 static void      xptasync(struct cam_periph *periph,
 683                           u_int32_t code, cam_path *path);
 684 #endif
 685 static dev_match_ret    xptbusmatch(struct dev_match_pattern *patterns,
 686                                     int num_patterns, struct cam_eb *bus);
 687 static dev_match_ret    xptdevicematch(struct dev_match_pattern *patterns,
 688                                        int num_patterns, struct cam_ed *device);
 689 static dev_match_ret    xptperiphmatch(struct dev_match_pattern *patterns,
 690                                        int num_patterns,
 691                                        struct cam_periph *periph);
 692 static xpt_busfunc_t    xptedtbusfunc;
 693 static xpt_targetfunc_t xptedttargetfunc;
 694 static xpt_devicefunc_t xptedtdevicefunc;
 695 static xpt_periphfunc_t xptedtperiphfunc;
 696 static xpt_pdrvfunc_t   xptplistpdrvfunc;
 697 static xpt_periphfunc_t xptplistperiphfunc;
 698 static int              xptedtmatch(struct ccb_dev_match *cdm);
 699 static int              xptperiphlistmatch(struct ccb_dev_match *cdm);
 700 static int              xptbustraverse(struct cam_eb *start_bus,
 701                                        xpt_busfunc_t *tr_func, void *arg);
 702 static int              xpttargettraverse(struct cam_eb *bus,
 703                                           struct cam_et *start_target,
 704                                           xpt_targetfunc_t *tr_func, void *arg);
 705 static int              xptdevicetraverse(struct cam_et *target,
 706                                           struct cam_ed *start_device,
 707                                           xpt_devicefunc_t *tr_func, void *arg);
 708 static int              xptperiphtraverse(struct cam_ed *device,
 709                                           struct cam_periph *start_periph,
 710                                           xpt_periphfunc_t *tr_func, void *arg);
 711 static int              xptpdrvtraverse(struct periph_driver **start_pdrv,
 712                                         xpt_pdrvfunc_t *tr_func, void *arg);
 713 static int              xptpdperiphtraverse(struct periph_driver **pdrv,
 714                                             struct cam_periph *start_periph,
 715                                             xpt_periphfunc_t *tr_func,
 716                                             void *arg);
 717 static xpt_busfunc_t    xptdefbusfunc;
 718 static xpt_targetfunc_t xptdeftargetfunc;
 719 static xpt_devicefunc_t xptdefdevicefunc;
 720 static xpt_periphfunc_t xptdefperiphfunc;
 721 static int              xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg);
 722 #ifdef notusedyet
 723 static int              xpt_for_all_targets(xpt_targetfunc_t *tr_func,
 724                                             void *arg);
 725 #endif
 726 static int              xpt_for_all_devices(xpt_devicefunc_t *tr_func,
 727                                             void *arg);
 728 #ifdef notusedyet
 729 static int              xpt_for_all_periphs(xpt_periphfunc_t *tr_func,
 730                                             void *arg);
 731 #endif
 732 static xpt_devicefunc_t xptsetasyncfunc;
 733 static xpt_busfunc_t    xptsetasyncbusfunc;
 734 static cam_status       xptregister(struct cam_periph *periph,
 735                                     void *arg);
 736 static cam_status       proberegister(struct cam_periph *periph,
 737                                       void *arg);
 738 static void      probeschedule(struct cam_periph *probe_periph);
 739 static void      probestart(struct cam_periph *periph, union ccb *start_ccb);
 740 static void      proberequestdefaultnegotiation(struct cam_periph *periph);
 741 static void      probedone(struct cam_periph *periph, union ccb *done_ccb);
 742 static void      probecleanup(struct cam_periph *periph);
 743 static void      xpt_find_quirk(struct cam_ed *device);
 744 static void      xpt_set_transfer_settings(struct ccb_trans_settings *cts,
 745                                            struct cam_ed *device,
 746                                            int async_update);
 747 static void      xpt_toggle_tags(struct cam_path *path);
 748 static void      xpt_start_tags(struct cam_path *path);
 749 static __inline int xpt_schedule_dev_allocq(struct cam_eb *bus,
 750                                             struct cam_ed *dev);
 751 static __inline int xpt_schedule_dev_sendq(struct cam_eb *bus,
 752                                            struct cam_ed *dev);
 753 static __inline int periph_is_queued(struct cam_periph *periph);
 754 static __inline int device_is_alloc_queued(struct cam_ed *device);
 755 static __inline int device_is_send_queued(struct cam_ed *device);
 756 static __inline int dev_allocq_is_runnable(struct cam_devq *devq);
 757
 758 static __inline int
 759 xpt_schedule_dev_allocq(struct cam_eb *bus, struct cam_ed *dev)
 760 {
 761         int retval;
 762
 763         if (dev->ccbq.devq_openings > 0) {
 764                 if ((dev->flags & CAM_DEV_RESIZE_QUEUE_NEEDED) != 0) {
 765                         cam_ccbq_resize(&dev->ccbq,
 766                                         dev->ccbq.dev_openings
 767                                         + dev->ccbq.dev_active);
 768                         dev->flags &= ~CAM_DEV_RESIZE_QUEUE_NEEDED;
 769                 }
 770                 /*
 771                  * The priority of a device waiting for CCB resources
 772                  * is that of the the highest priority peripheral driver
 773                  * enqueued.
 774                  */
 775                 retval = xpt_schedule_dev(&bus->sim->devq->alloc_queue,
 776                                           &dev->alloc_ccb_entry.pinfo,
 777                                           CAMQ_GET_HEAD(&dev->drvq)->priority);
 778         } else {
 779                 retval = 0;
 780         }
 781
 782         return (retval);
 783 }
 784
 785 static __inline int
 786 xpt_schedule_dev_sendq(struct cam_eb *bus, struct cam_ed *dev)
 787 {
 788         int     retval;
 789
 790         if (dev->ccbq.dev_openings > 0) {
 791                 /*
 792                  * The priority of a device waiting for controller
 793                  * resources is that of the the highest priority CCB
 794                  * enqueued.
 795                  */
 796                 retval =
 797                     xpt_schedule_dev(&bus->sim->devq->send_queue,
 798                                      &dev->send_ccb_entry.pinfo,
 799                                      CAMQ_GET_HEAD(&dev->ccbq.queue)->priority);
 800         } else {
 801                 retval = 0;
 802         }
 803         return (retval);
 804 }
 805
 806 static __inline int
 807 periph_is_queued(struct cam_periph *periph)
 808 {
 809         return (periph->pinfo.index != CAM_UNQUEUED_INDEX);
 810 }
 811
 812 static __inline int
 813 device_is_alloc_queued(struct cam_ed *device)
 814 {
 815         return (device->alloc_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX);
 816 }
 817
 818 static __inline int
 819 device_is_send_queued(struct cam_ed *device)
 820 {
 821         return (device->send_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX);
 822 }
 823
 824 static __inline int
 825 dev_allocq_is_runnable(struct cam_devq *devq)
 826 {
 827         /*
 828          * Have work to do.
 829          * Have space to do more work.
 830          * Allowed to do work.
 831          */
 832         return ((devq->alloc_queue.qfrozen_cnt == 0)
 833              && (devq->alloc_queue.entries > 0)
 834              && (devq->alloc_openings > 0));
 835 }
 836
 837 static void
 838 xpt_periph_init()
 839 {
 840         make_dev(&xpt_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600, "xpt0");
 841 }
 842
 843 static void
 844 probe_periph_init()
 845 {
 846 }
 847
 848
 849 static void
 850 xptdone(struct cam_periph *periph, union ccb *done_ccb)
 851 {
 852         /* Caller will release the CCB */
 853         wakeup(&done_ccb->ccb_h.cbfcnp);
 854 }
 855
 856 static int
 857 xptopen(dev_t dev, int flags, int fmt, struct proc *p)
 858 {
 859         int unit;
 860
 861         unit = minor(dev) & 0xff;
 862
 863         /*
 864          * Only allow read-write access.
 865          */
 866         if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0))
 867                 return(EPERM);
 868
 869         /*
 870          * We don't allow nonblocking access.
 871          */
 872         if ((flags & O_NONBLOCK) != 0) {
 873                 printf("xpt%d: can't do nonblocking accesss\n", unit);
 874                 return(ENODEV);
 875         }
 876
 877         /*
 878          * We only have one transport layer right now.  If someone accesses
 879          * us via something other than minor number 1, point out their
 880          * mistake.
 881          */
 882         if (unit != 0) {
 883                 printf("xptopen: got invalid xpt unit %d\n", unit);
 884                 return(ENXIO);
 885         }
 886
 887         /* Mark ourselves open */
 888         xsoftc.flags |= XPT_FLAG_OPEN;
 889
 890         return(0);
 891 }
 892
 893 static int
 894 xptclose(dev_t dev, int flag, int fmt, struct proc *p)
 895 {
 896         int unit;
 897
 898         unit = minor(dev) & 0xff;
 899
 900         /*
 901          * We only have one transport layer right now.  If someone accesses
 902          * us via something other than minor number 1, point out their
 903          * mistake.
 904          */
 905         if (unit != 0) {
 906                 printf("xptclose: got invalid xpt unit %d\n", unit);
 907                 return(ENXIO);
 908         }
 909
 910         /* Mark ourselves closed */
 911         xsoftc.flags &= ~XPT_FLAG_OPEN;
 912
 913         return(0);
 914 }
 915
 916 static int
 917 xptioctl(dev_t dev, u_long cmd, caddr_t addr, int flag, struct proc *p)
 918 {
 919         int unit, error;
 920
 921         error = 0;
 922         unit = minor(dev) & 0xff;
 923
 924         /*
 925          * We only have one transport layer right now.  If someone accesses
 926          * us via something other than minor number 1, point out their
 927          * mistake.
 928          */
 929         if (unit != 0) {
 930                 printf("xptioctl: got invalid xpt unit %d\n", unit);
 931                 return(ENXIO);
 932         }
 933
 934         switch(cmd) {
 935         /*
 936          * For the transport layer CAMIOCOMMAND ioctl, we really only want
 937          * to accept CCB types that don't quite make sense to send through a
 938          * passthrough driver.
 939          */
 940         case CAMIOCOMMAND: {
 941                 union ccb *ccb;
 942                 union ccb *inccb;
 943
 944                 inccb = (union ccb *)addr;
 945
 946                 switch(inccb->ccb_h.func_code) {
 947                 case XPT_SCAN_BUS:
 948                 case XPT_RESET_BUS:
 949                         if ((inccb->ccb_h.target_id != CAM_TARGET_WILDCARD)
 950                          || (inccb->ccb_h.target_lun != CAM_LUN_WILDCARD)) {
 951                                 error = EINVAL;
 952                                 break;
 953                         }
 954                         /* FALLTHROUGH */
 955                 case XPT_SCAN_LUN:
 956
 957                         ccb = xpt_alloc_ccb();
 958
 959                         /*
 960                          * Create a path using the bus, target, and lun the
 961                          * user passed in.
 962                          */
 963                         if (xpt_create_path(&ccb->ccb_h.path, xpt_periph,
 964                                             inccb->ccb_h.path_id,
 965                                             inccb->ccb_h.target_id,
 966                                             inccb->ccb_h.target_lun) !=
 967                                             CAM_REQ_CMP){
 968                                 error = EINVAL;
 969                                 xpt_free_ccb(ccb);
 970                                 break;
 971                         }
 972                         /* Ensure all of our fields are correct */
 973                         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path,
 974                                       inccb->ccb_h.pinfo.priority);
 975                         xpt_merge_ccb(ccb, inccb);
 976                         ccb->ccb_h.cbfcnp = xptdone;
 977                         cam_periph_runccb(ccb, NULL, 0, 0, NULL);
 978                         bcopy(ccb, inccb, sizeof(union ccb));
 979                         xpt_free_path(ccb->ccb_h.path);
 980                         xpt_free_ccb(ccb);
 981                         break;
 982
 983                 case XPT_DEBUG: {
 984                         union ccb ccb;
 985
 986                         /*
 987                          * This is an immediate CCB, so it's okay to
 988                          * allocate it on the stack.
 989                          */
 990
 991                         /*
 992                          * Create a path using the bus, target, and lun the
 993                          * user passed in.
 994                          */
 995                         if (xpt_create_path(&ccb.ccb_h.path, xpt_periph,
 996                                             inccb->ccb_h.path_id,
 997                                             inccb->ccb_h.target_id,
 998                                             inccb->ccb_h.target_lun) !=
 999                                             CAM_REQ_CMP){
1000                                 error = EINVAL;
1001                                 break;
1002                         }
1003                         /* Ensure all of our fields are correct */
1004                         xpt_setup_ccb(&ccb.ccb_h, ccb.ccb_h.path,
1005                                       inccb->ccb_h.pinfo.priority);
1006                         xpt_merge_ccb(&ccb, inccb);
1007                         ccb.ccb_h.cbfcnp = xptdone;
1008                         xpt_action(&ccb);
1009                         bcopy(&ccb, inccb, sizeof(union ccb));
1010                         xpt_free_path(ccb.ccb_h.path);
1011                         break;
1012
1013                 }
1014                 case XPT_DEV_MATCH: {
1015                         struct cam_periph_map_info mapinfo;
1016                         struct cam_path *old_path;
1017
1018                         /*
1019                          * We can't deal with physical addresses for this
1020                          * type of transaction.
1021                          */
1022                         if (inccb->ccb_h.flags & CAM_DATA_PHYS) {
1023                                 error = EINVAL;
1024                                 break;
1025                         }
1026
1027                         /*
1028                          * Save this in case the caller had it set to
1029                          * something in particular.
1030                          */
1031                         old_path = inccb->ccb_h.path;
1032
1033                         /*
1034                          * We really don't need a path for the matching
1035                          * code.  The path is needed because of the
1036                          * debugging statements in xpt_action().  They
1037                          * assume that the CCB has a valid path.
1038                          */
1039                         inccb->ccb_h.path = xpt_periph->path;
1040
1041                         bzero(&mapinfo, sizeof(mapinfo));
1042
1043                         /*
1044                          * Map the pattern and match buffers into kernel
1045                          * virtual address space.
1046                          */
1047                         error = cam_periph_mapmem(inccb, &mapinfo);
1048
1049                         if (error) {
1050                                 inccb->ccb_h.path = old_path;
1051                                 break;
1052                         }
1053
1054                         /*
1055                          * This is an immediate CCB, we can send it on directly.
1056                          */
1057                         xpt_action(inccb);
1058
1059                         /*
1060                          * Map the buffers back into user space.
1061                          */
1062                         cam_periph_unmapmem(inccb, &mapinfo);
1063
1064                         inccb->ccb_h.path = old_path;
1065
1066                         error = 0;
1067                         break;
1068                 }
1069                 default:
1070                         error = EINVAL;
1071                         break;
1072                 }
1073                 break;
1074         }
1075         /*
1076          * This is the getpassthru ioctl. It takes a XPT_GDEVLIST ccb as input,
1077          * with the periphal driver name and unit name filled in.  The other
1078          * fields don't really matter as input.  The passthrough driver name
1079          * ("pass"), and unit number are passed back in the ccb.  The current
1080          * device generation number, and the index into the device peripheral
1081          * driver list, and the status are also passed back.  Note that
1082          * since we do everything in one pass, unlike the XPT_GDEVLIST ccb,
1083          * we never return a status of CAM_GDEVLIST_LIST_CHANGED.  It is
1084          * (or rather should be) impossible for the device peripheral driver
1085          * list to change since we look at the whole thing in one pass, and
1086          * we do it with splcam protection.
1087          *
1088          */
1089         case CAMGETPASSTHRU: {
1090                 union ccb *ccb;
1091                 struct cam_periph *periph;
1092                 struct periph_driver **p_drv;
1093                 char   *name;
1094                 int unit;
1095                 int cur_generation;
1096                 int base_periph_found;
1097                 int splbreaknum;
1098                 int s;
1099
1100                 ccb = (union ccb *)addr;
1101                 unit = ccb->cgdl.unit_number;
1102                 name = ccb->cgdl.periph_name;
1103                 /*
1104                  * Every 100 devices, we want to drop our spl protection to
1105                  * give the software interrupt handler a chance to run.
1106                  * Most systems won't run into this check, but this should
1107                  * avoid starvation in the software interrupt handler in
1108                  * large systems.
1109                  */
1110                 splbreaknum = 100;
1111
1112                 ccb = (union ccb *)addr;
1113
1114                 base_periph_found = 0;
1115
1116                 /*
1117                  * Sanity check -- make sure we don't get a null peripheral
1118                  * driver name.
1119                  */
1120                 if (*ccb->cgdl.periph_name == '\0') {
1121                         error = EINVAL;
1122                         break;
1123                 }
1124
1125                 /* Keep the list from changing while we traverse it */
1126                 s = splcam();
1127 ptstartover:
1128                 cur_generation = xsoftc.generation;
1129
1130                 /* first find our driver in the list of drivers */
1131                 for (p_drv = (struct periph_driver **)periphdriver_set.ls_items;
1132                      *p_drv != NULL; p_drv++)
1133                         if (strcmp((*p_drv)->driver_name, name) == 0)
1134                                 break;
1135
1136                 if (*p_drv == NULL) {
1137                         splx(s);
1138                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1139                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
1140                         *ccb->cgdl.periph_name = '\0';
1141                         ccb->cgdl.unit_number = 0;
1142                         error = ENOENT;
1143                         break;
1144                 }
1145
1146                 /*
1147                  * Run through every peripheral instance of this driver
1148                  * and check to see whether it matches the unit passed
1149                  * in by the user.  If it does, get out of the loops and
1150                  * find the passthrough driver associated with that
1151                  * peripheral driver.
1152                  */
1153                 for (periph = TAILQ_FIRST(&(*p_drv)->units); periph != NULL;
1154                      periph = TAILQ_NEXT(periph, unit_links)) {
1155
1156                         if (periph->unit_number == unit) {
1157                                 break;
1158                         } else if (--splbreaknum == 0) {
1159                                 splx(s);
1160                                 s = splcam();
1161                                 splbreaknum = 100;
1162                                 if (cur_generation != xsoftc.generation)
1163                                        goto ptstartover;
1164                         }
1165                 }
1166                 /*
1167                  * If we found the peripheral driver that the user passed
1168                  * in, go through all of the peripheral drivers for that
1169                  * particular device and look for a passthrough driver.
1170                  */
1171                 if (periph != NULL) {
1172                         struct cam_ed *device;
1173                         int i;
1174
1175                         base_periph_found = 1;
1176                         device = periph->path->device;
1177                         for (i = 0, periph = device->periphs.slh_first;
1178                              periph != NULL;
1179                              periph = periph->periph_links.sle_next, i++) {
1180                                 /*
1181                                  * Check to see whether we have a
1182                                  * passthrough device or not.
1183                                  */
1184                                 if (strcmp(periph->periph_name, "pass") == 0) {
1185                                         /*
1186                                          * Fill in the getdevlist fields.
1187                                          */
1188                                         strcpy(ccb->cgdl.periph_name,
1189                                                periph->periph_name);
1190                                         ccb->cgdl.unit_number =
1191                                                 periph->unit_number;
1192                                         if (periph->periph_links.sle_next)
1193                                                 ccb->cgdl.status =
1194                                                         CAM_GDEVLIST_MORE_DEVS;
1195                                         else
1196                                                 ccb->cgdl.status =
1197                                                        CAM_GDEVLIST_LAST_DEVICE;
1198                                         ccb->cgdl.generation =
1199                                                 device->generation;
1200                                         ccb->cgdl.index = i;
1201                                         /*
1202                                          * Fill in some CCB header fields
1203                                          * that the user may want.
1204                                          */
1205                                         ccb->ccb_h.path_id =
1206                                                 periph->path->bus->path_id;
1207                                         ccb->ccb_h.target_id =
1208                                                 periph->path->target->target_id;
1209                                         ccb->ccb_h.target_lun =
1210                                                 periph->path->device->lun_id;
1211                                         ccb->ccb_h.status = CAM_REQ_CMP;
1212                                         break;
1213                                 }
1214                         }
1215                 }
1216
1217                 /*
1218                  * If the periph is null here, one of two things has
1219                  * happened.  The first possibility is that we couldn't
1220                  * find the unit number of the particular peripheral driver
1221                  * that the user is asking about.  e.g. the user asks for
1222                  * the passthrough driver for "da11".  We find the list of
1223                  * "da" peripherals all right, but there is no unit 11.
1224                  * The other possibility is that we went through the list
1225                  * of peripheral drivers attached to the device structure,
1226                  * but didn't find one with the name "pass".  Either way,
1227                  * we return ENOENT, since we couldn't find something.
1228                  */
1229                 if (periph == NULL) {
1230                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1231                         ccb->cgdl.status = CAM_GDEVLIST_ERROR;
1232                         *ccb->cgdl.periph_name = '\0';
1233                         ccb->cgdl.unit_number = 0;
1234                         error = ENOENT;
1235                         /*
1236                          * It is unfortunate that this is even necessary,
1237                          * but there are many, many clueless users out there.
1238                          * If this is true, the user is looking for the
1239                          * passthrough driver, but doesn't have one in his
1240                          * kernel.
1241                          */
1242                         if (base_periph_found == 1) {
1243                                 printf("xptioctl: pass driver is not in the "
1244                                        "kernel\n");
1245                                 printf("xptioctl: put \"device pass0\" in "
1246                                        "your kernel config file\n");
1247                         }
1248                 }
1249                 splx(s);
1250                 break;
1251                 }
1252         default:
1253                 error = ENOTTY;
1254                 break;
1255         }
1256
1257         return(error);
1258 }
1259
1260 /* Functions accessed by the peripheral drivers */
1261 static void
1262 xpt_init(dummy)
1263         void *dummy;
1264 {
1265         struct cam_sim *xpt_sim;
1266         struct cam_path *path;
1267         struct cam_devq *devq;
1268         cam_status status;
1269
1270         TAILQ_INIT(&xpt_busses);
1271         TAILQ_INIT(&cam_bioq);
1272         TAILQ_INIT(&cam_netq);
1273         SLIST_INIT(&ccb_freeq);
1274         STAILQ_INIT(&highpowerq);
1275
1276         /*
1277          * The xpt layer is, itself, the equivelent of a SIM.
1278          * Allow 16 ccbs in the ccb pool for it.  This should
1279          * give decent parallelism when we probe busses and
1280          * perform other XPT functions.
1281          */
1282         devq = cam_simq_alloc(16);
1283         xpt_sim = cam_sim_alloc(xptaction,
1284                                 xptpoll,
1285                                 "xpt",
1286                                 /*softc*/NULL,
1287                                 /*unit*/0,
1288                                 /*max_dev_transactions*/0,
1289                                 /*max_tagged_dev_transactions*/0,
1290                                 devq);
1291         xpt_max_ccbs = 16;
1292
1293         xpt_bus_register(xpt_sim, /*bus #*/0);
1294
1295         /*
1296          * Looking at the XPT from the SIM layer, the XPT is
1297          * the equivelent of a peripheral driver.  Allocate
1298          * a peripheral driver entry for us.
1299          */
1300         if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
1301                                       CAM_TARGET_WILDCARD,
1302                                       CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
1303                 printf("xpt_init: xpt_create_path failed with status %#x,"
1304                        " failing attach\n", status);
1305                 return;
1306         }
1307
1308         cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
1309                          path, NULL, 0, NULL);
1310         xpt_free_path(path);
1311
1312         xpt_sim->softc = xpt_periph;
1313
1314         /*
1315          * Register a callback for when interrupts are enabled.
1316          */
1317         xpt_config_hook =
1318             (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
1319                                               M_TEMP, M_NOWAIT);
1320         if (xpt_config_hook == NULL) {
1321                 printf("xpt_init: Cannot malloc config hook "
1322                        "- failing attach\n");
1323                 return;
1324         }
1325         bzero(xpt_config_hook, sizeof(*xpt_config_hook));
1326
1327         xpt_config_hook->ich_func = xpt_config;
1328         if (config_intrhook_establish(xpt_config_hook) != 0) {
1329                 free (xpt_config_hook, M_TEMP);
1330                 printf("xpt_init: config_intrhook_establish failed "
1331                        "- failing attach\n");
1332         }
1333
1334         /* Install our software interrupt handlers */
1335         register_swi(SWI_CAMNET, swi_camnet);
1336         register_swi(SWI_CAMBIO, swi_cambio);
1337 }
1338
1339 static cam_status
1340 xptregister(struct cam_periph *periph, void *arg)
1341 {
1342         if (periph == NULL) {
1343                 printf("xptregister: periph was NULL!!\n");
1344                 return(CAM_REQ_CMP_ERR);
1345         }
1346
1347         periph->softc = NULL;
1348
1349         xpt_periph = periph;
1350
1351         return(CAM_REQ_CMP);
1352 }
1353
1354 int32_t
1355 xpt_add_periph(struct cam_periph *periph)
1356 {
1357         struct cam_ed *device;
1358         int32_t  status;
1359         struct periph_list *periph_head;
1360
1361         device = periph->path->device;
1362
1363         periph_head = &device->periphs;
1364
1365         status = CAM_REQ_CMP;
1366
1367         if (device != NULL) {
1368                 int s;
1369
1370                 /*
1371                  * Make room for this peripheral
1372                  * so it will fit in the queue
1373                  * when it's scheduled to run
1374                  */
1375                 s = splsoftcam();
1376                 status = camq_resize(&device->drvq,
1377                                      device->drvq.array_size + 1);
1378
1379                 device->generation++;
1380
1381                 SLIST_INSERT_HEAD(periph_head, periph, periph_links);
1382
1383                 splx(s);
1384         }
1385
1386         xsoftc.generation++;
1387
1388         return (status);
1389 }
1390
1391 void
1392 xpt_remove_periph(struct cam_periph *periph)
1393 {
1394         struct cam_ed *device;
1395
1396         device = periph->path->device;
1397
1398         if (device != NULL) {
1399                 int s;
1400                 struct periph_list *periph_head;
1401
1402                 periph_head = &device->periphs;
1403
1404                 /* Release the slot for this peripheral */
1405                 s = splsoftcam();
1406                 camq_resize(&device->drvq, device->drvq.array_size - 1);
1407
1408                 device->generation++;
1409
1410                 SLIST_REMOVE(periph_head, periph, cam_periph, periph_links);
1411
1412                 splx(s);
1413         }
1414
1415         xsoftc.generation++;
1416
1417 }
1418
1419 void
1420 xpt_announce_periph(struct cam_periph *periph, char *announce_string)
1421 {
1422         int s;
1423         u_int mb;
1424         struct cam_path *path;
1425         struct ccb_trans_settings cts;
1426
1427         path = periph->path;
1428         /*
1429          * To ensure that this is printed in one piece,
1430          * mask out CAM interrupts.
1431          */
1432         s = splsoftcam();
1433         printf("%s%d at %s%d bus %d target %d lun %d\n",
1434                periph->periph_name, periph->unit_number,
1435                path->bus->sim->sim_name,
1436                path->bus->sim->unit_number,
1437                path->bus->sim->bus_id,
1438                path->target->target_id,
1439                path->device->lun_id);
1440         printf("%s%d: ", periph->periph_name, periph->unit_number);
1441         scsi_print_inquiry(&path->device->inq_data);
1442         if ((bootverbose)
1443          && (path->device->serial_num_len > 0)) {
1444                 /* Don't wrap the screen  - print only the first 60 chars */
1445                 printf("%s%d: Serial Number %.60s\n", periph->periph_name,
1446                        periph->unit_number, path->device->serial_num);
1447         }
1448         xpt_setup_ccb(&cts.ccb_h, path, /*priority*/1);
1449         cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
1450         cts.flags = CCB_TRANS_CURRENT_SETTINGS;
1451         xpt_action((union ccb*)&cts);
1452         if (cts.ccb_h.status == CAM_REQ_CMP) {
1453                 u_int speed;
1454                 u_int freq;
1455
1456                 if ((cts.valid & CCB_TRANS_SYNC_OFFSET_VALID) != 0
1457                   && cts.sync_offset != 0) {
1458                         freq = scsi_calc_syncsrate(cts.sync_period);
1459                         speed = freq;
1460                 } else {
1461                         struct ccb_pathinq cpi;
1462
1463                         /* Ask the SIM for its base transfer speed */
1464                         xpt_setup_ccb(&cpi.ccb_h, path, /*priority*/1);
1465                         cpi.ccb_h.func_code = XPT_PATH_INQ;
1466                         xpt_action((union ccb *)&cpi);
1467
1468                         speed = cpi.base_transfer_speed;
1469                         freq = 0;
1470                 }
1471                 if ((cts.valid & CCB_TRANS_BUS_WIDTH_VALID) != 0)
1472                         speed *= (0x01 << cts.bus_width);
1473                 mb = speed / 1000;
1474                 if (mb > 0)
1475                         printf("%s%d: %d.%03dMB/s transfers",
1476                                periph->periph_name, periph->unit_number,
1477                                mb, speed % 1000);
1478                 else
1479                         printf("%s%d: %dKB/s transfers", periph->periph_name,
1480                                periph->unit_number, speed);
1481                 if ((cts.valid & CCB_TRANS_SYNC_OFFSET_VALID) != 0
1482                  && cts.sync_offset != 0) {
1483                         printf(" (%d.%03dMHz, offset %d", freq / 1000,
1484                                freq % 1000, cts.sync_offset);
1485                 }
1486                 if ((cts.valid & CCB_TRANS_BUS_WIDTH_VALID) != 0
1487                  && cts.bus_width > 0) {
1488                         if ((cts.valid & CCB_TRANS_SYNC_OFFSET_VALID) != 0
1489                          && cts.sync_offset != 0) {
1490                                 printf(", ");
1491                         } else {
1492                                 printf(" (");
1493                         }
1494                         printf("%dbit)", 8 * (0x01 << cts.bus_width));
1495                 } else if ((cts.valid & CCB_TRANS_SYNC_OFFSET_VALID) != 0
1496                         && cts.sync_offset != 0) {
1497                         printf(")");
1498                 }
1499
1500                 if (path->device->inq_flags & SID_CmdQue
1501                  || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1502                         printf(", Tagged Queueing Enabled");
1503                 }
1504
1505                 printf("\n");
1506         } else if (path->device->inq_flags & SID_CmdQue
1507                 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
1508                 printf("%s%d: Tagged Queueing Enabled\n",
1509                        periph->periph_name, periph->unit_number);
1510         }
1511
1512         /*
1513          * We only want to print the caller's announce string if they've
1514          * passed one in..
1515          */
1516         if (announce_string != NULL)
1517                 printf("%s%d: %s\n", periph->periph_name,
1518                        periph->unit_number, announce_string);
1519         splx(s);
1520 }
1521
1522
1523 static dev_match_ret
1524 xptbusmatch(struct dev_match_pattern *patterns, int num_patterns,
1525             struct cam_eb *bus)
1526 {
1527         dev_match_ret retval;
1528         int i;
1529
1530         retval = DM_RET_NONE;
1531
1532         /*
1533          * If we aren't given something to match against, that's an error.
1534          */
1535         if (bus == NULL)
1536                 return(DM_RET_ERROR);
1537
1538         /*
1539          * If there are no match entries, then this bus matches no
1540          * matter what.
1541          */
1542         if ((patterns == NULL) || (num_patterns == 0))
1543                 return(DM_RET_DESCEND | DM_RET_COPY);
1544
1545         for (i = 0; i < num_patterns; i++) {
1546                 struct bus_match_pattern *cur_pattern;
1547
1548                 /*
1549                  * If the pattern in question isn't for a bus node, we
1550                  * aren't interested.  However, we do indicate to the
1551                  * calling routine that we should continue descending the
1552                  * tree, since the user wants to match against lower-level
1553                  * EDT elements.
1554                  */
1555                 if (patterns[i].type != DEV_MATCH_BUS) {
1556                         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1557                                 retval |= DM_RET_DESCEND;
1558                         continue;
1559                 }
1560
1561                 cur_pattern = &patterns[i].pattern.bus_pattern;
1562
1563                 /*
1564                  * If they want to match any bus node, we give them any
1565                  * device node.
1566                  */
1567                 if (cur_pattern->flags == BUS_MATCH_ANY) {
1568                         /* set the copy flag */
1569                         retval |= DM_RET_COPY;
1570
1571                         /*
1572                          * If we've already decided on an action, go ahead
1573                          * and return.
1574                          */
1575                         if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1576                                 return(retval);
1577                 }
1578
1579                 /*
1580                  * Not sure why someone would do this...
1581                  */
1582                 if (cur_pattern->flags == BUS_MATCH_NONE)
1583                         continue;
1584
1585                 if (((cur_pattern->flags & BUS_MATCH_PATH) != 0)
1586                  && (cur_pattern->path_id != bus->path_id))
1587                         continue;
1588
1589                 if (((cur_pattern->flags & BUS_MATCH_BUS_ID) != 0)
1590                  && (cur_pattern->bus_id != bus->sim->bus_id))
1591                         continue;
1592
1593                 if (((cur_pattern->flags & BUS_MATCH_UNIT) != 0)
1594                  && (cur_pattern->unit_number != bus->sim->unit_number))
1595                         continue;
1596
1597                 if (((cur_pattern->flags & BUS_MATCH_NAME) != 0)
1598                  && (strncmp(cur_pattern->dev_name, bus->sim->sim_name,
1599                              DEV_IDLEN) != 0))
1600                         continue;
1601
1602                 /*
1603                  * If we get to this point, the user definitely wants
1604                  * information on this bus.  So tell the caller to copy the
1605                  * data out.
1606                  */
1607                 retval |= DM_RET_COPY;
1608
1609                 /*
1610                  * If the return action has been set to descend, then we
1611                  * know that we've already seen a non-bus matching
1612                  * expression, therefore we need to further descend the tree.
1613                  * This won't change by continuing around the loop, so we
1614                  * go ahead and return.  If we haven't seen a non-bus
1615                  * matching expression, we keep going around the loop until
1616                  * we exhaust the matching expressions.  We'll set the stop
1617                  * flag once we fall out of the loop.
1618                  */
1619                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1620                         return(retval);
1621         }
1622
1623         /*
1624          * If the return action hasn't been set to descend yet, that means
1625          * we haven't seen anything other than bus matching patterns.  So
1626          * tell the caller to stop descending the tree -- the user doesn't
1627          * want to match against lower level tree elements.
1628          */
1629         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1630                 retval |= DM_RET_STOP;
1631
1632         return(retval);
1633 }
1634
1635 static dev_match_ret
1636 xptdevicematch(struct dev_match_pattern *patterns, int num_patterns,
1637                struct cam_ed *device)
1638 {
1639         dev_match_ret retval;
1640         int i;
1641
1642         retval = DM_RET_NONE;
1643
1644         /*
1645          * If we aren't given something to match against, that's an error.
1646          */
1647         if (device == NULL)
1648                 return(DM_RET_ERROR);
1649
1650         /*
1651          * If there are no match entries, then this device matches no
1652          * matter what.
1653          */
1654         if ((patterns == NULL) || (patterns == 0))
1655                 return(DM_RET_DESCEND | DM_RET_COPY);
1656
1657         for (i = 0; i < num_patterns; i++) {
1658                 struct device_match_pattern *cur_pattern;
1659
1660                 /*
1661                  * If the pattern in question isn't for a device node, we
1662                  * aren't interested.
1663                  */
1664                 if (patterns[i].type != DEV_MATCH_DEVICE) {
1665                         if ((patterns[i].type == DEV_MATCH_PERIPH)
1666                          && ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE))
1667                                 retval |= DM_RET_DESCEND;
1668                         continue;
1669                 }
1670
1671                 cur_pattern = &patterns[i].pattern.device_pattern;
1672
1673                 /*
1674                  * If they want to match any device node, we give them any
1675                  * device node.
1676                  */
1677                 if (cur_pattern->flags == DEV_MATCH_ANY) {
1678                         /* set the copy flag */
1679                         retval |= DM_RET_COPY;
1680
1681
1682                         /*
1683                          * If we've already decided on an action, go ahead
1684                          * and return.
1685                          */
1686                         if ((retval & DM_RET_ACTION_MASK) != DM_RET_NONE)
1687                                 return(retval);
1688                 }
1689
1690                 /*
1691                  * Not sure why someone would do this...
1692                  */
1693                 if (cur_pattern->flags == DEV_MATCH_NONE)
1694                         continue;
1695
1696                 if (((cur_pattern->flags & DEV_MATCH_PATH) != 0)
1697                  && (cur_pattern->path_id != device->target->bus->path_id))
1698                         continue;
1699
1700                 if (((cur_pattern->flags & DEV_MATCH_TARGET) != 0)
1701                  && (cur_pattern->target_id != device->target->target_id))
1702                         continue;
1703
1704                 if (((cur_pattern->flags & DEV_MATCH_LUN) != 0)
1705                  && (cur_pattern->target_lun != device->lun_id))
1706                         continue;
1707
1708                 if (((cur_pattern->flags & DEV_MATCH_INQUIRY) != 0)
1709                  && (cam_quirkmatch((caddr_t)&device->inq_data,
1710                                     (caddr_t)&cur_pattern->inq_pat,
1711                                     1, sizeof(cur_pattern->inq_pat),
1712                                     scsi_static_inquiry_match) == NULL))
1713                         continue;
1714
1715                 /*
1716                  * If we get to this point, the user definitely wants
1717                  * information on this device.  So tell the caller to copy
1718                  * the data out.
1719                  */
1720                 retval |= DM_RET_COPY;
1721
1722                 /*
1723                  * If the return action has been set to descend, then we
1724                  * know that we've already seen a peripheral matching
1725                  * expression, therefore we need to further descend the tree.
1726                  * This won't change by continuing around the loop, so we
1727                  * go ahead and return.  If we haven't seen a peripheral
1728                  * matching expression, we keep going around the loop until
1729                  * we exhaust the matching expressions.  We'll set the stop
1730                  * flag once we fall out of the loop.
1731                  */
1732                 if ((retval & DM_RET_ACTION_MASK) == DM_RET_DESCEND)
1733                         return(retval);
1734         }
1735
1736         /*
1737          * If the return action hasn't been set to descend yet, that means
1738          * we haven't seen any peripheral matching patterns.  So tell the
1739          * caller to stop descending the tree -- the user doesn't want to
1740          * match against lower level tree elements.
1741          */
1742         if ((retval & DM_RET_ACTION_MASK) == DM_RET_NONE)
1743                 retval |= DM_RET_STOP;
1744
1745         return(retval);
1746 }
1747
1748 /*
1749  * Match a single peripheral against any number of match patterns.
1750  */
1751 static dev_match_ret
1752 xptperiphmatch(struct dev_match_pattern *patterns, int num_patterns,
1753                struct cam_periph *periph)
1754 {
1755         dev_match_ret retval;
1756         int i;
1757
1758         /*
1759          * If we aren't given something to match against, that's an error.
1760          */
1761         if (periph == NULL)
1762                 return(DM_RET_ERROR);
1763
1764         /*
1765          * If there are no match entries, then this peripheral matches no
1766          * matter what.
1767          */
1768         if ((patterns == NULL) || (num_patterns == 0))
1769                 return(DM_RET_STOP | DM_RET_COPY);
1770
1771         /*
1772          * There aren't any nodes below a peripheral node, so there's no
1773          * reason to descend the tree any further.
1774          */
1775         retval = DM_RET_STOP;
1776
1777         for (i = 0; i < num_patterns; i++) {
1778                 struct periph_match_pattern *cur_pattern;
1779
1780                 /*
1781                  * If the pattern in question isn't for a peripheral, we
1782                  * aren't interested.
1783                  */
1784                 if (patterns[i].type != DEV_MATCH_PERIPH)
1785                         continue;
1786
1787                 cur_pattern = &patterns[i].pattern.periph_pattern;
1788
1789                 /*
1790                  * If they want to match on anything, then we will do so.
1791                  */
1792                 if (cur_pattern->flags == PERIPH_MATCH_ANY) {
1793                         /* set the copy flag */
1794                         retval |= DM_RET_COPY;
1795
1796                         /*
1797                          * We've already set the return action to stop,
1798                          * since there are no nodes below peripherals in
1799                          * the tree.
1800                          */
1801                         return(retval);
1802                 }
1803
1804                 /*
1805                  * Not sure why someone would do this...
1806                  */
1807                 if (cur_pattern->flags == PERIPH_MATCH_NONE)
1808                         continue;
1809
1810                 if (((cur_pattern->flags & PERIPH_MATCH_PATH) != 0)
1811                  && (cur_pattern->path_id != periph->path->bus->path_id))
1812                         continue;
1813
1814                 /*
1815                  * For the target and lun id's, we have to make sure the
1816                  * target and lun pointers aren't NULL.  The xpt peripheral
1817                  * has a wildcard target and device.
1818                  */
1819                 if (((cur_pattern->flags & PERIPH_MATCH_TARGET) != 0)
1820                  && ((periph->path->target == NULL)
1821                  ||(cur_pattern->target_id != periph->path->target->target_id)))
1822                         continue;
1823
1824                 if (((cur_pattern->flags & PERIPH_MATCH_LUN) != 0)
1825                  && ((periph->path->device == NULL)
1826                  || (cur_pattern->target_lun != periph->path->device->lun_id)))
1827                         continue;
1828
1829                 if (((cur_pattern->flags & PERIPH_MATCH_UNIT) != 0)
1830                  && (cur_pattern->unit_number != periph->unit_number))
1831                         continue;
1832
1833                 if (((cur_pattern->flags & PERIPH_MATCH_NAME) != 0)
1834                  && (strncmp(cur_pattern->periph_name, periph->periph_name,
1835                              DEV_IDLEN) != 0))
1836                         continue;
1837
1838                 /*
1839                  * If we get to this point, the user definitely wants
1840                  * information on this peripheral.  So tell the caller to
1841                  * copy the data out.
1842                  */
1843                 retval |= DM_RET_COPY;
1844
1845                 /*
1846                  * The return action has already been set to stop, since
1847                  * peripherals don't have any nodes below them in the EDT.
1848                  */
1849                 return(retval);
1850         }
1851
1852         /*
1853          * If we get to this point, the peripheral that was passed in
1854          * doesn't match any of the patterns.
1855          */
1856         return(retval);
1857 }
1858
1859 static int
1860 xptedtbusfunc(struct cam_eb *bus, void *arg)
1861 {
1862         struct ccb_dev_match *cdm;
1863         dev_match_ret retval;
1864
1865         cdm = (struct ccb_dev_match *)arg;
1866
1867         /*
1868          * If our position is for something deeper in the tree, that means
1869          * that we've already seen this node.  So, we keep going down.
1870          */
1871         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1872          && (cdm->pos.cookie.bus == bus)
1873          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1874          && (cdm->pos.cookie.target != NULL))
1875                 retval = DM_RET_DESCEND;
1876         else
1877                 retval = xptbusmatch(cdm->patterns, cdm->num_patterns, bus);
1878
1879         /*
1880          * If we got an error, bail out of the search.
1881          */
1882         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
1883                 cdm->status = CAM_DEV_MATCH_ERROR;
1884                 return(0);
1885         }
1886
1887         /*
1888          * If the copy flag is set, copy this bus out.
1889          */
1890         if (retval & DM_RET_COPY) {
1891                 int spaceleft, j;
1892
1893                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
1894                         sizeof(struct dev_match_result));
1895
1896                 /*
1897                  * If we don't have enough space to put in another
1898                  * match result, save our position and tell the
1899                  * user there are more devices to check.
1900                  */
1901                 if (spaceleft < sizeof(struct dev_match_result)) {
1902                         bzero(&cdm->pos, sizeof(cdm->pos));
1903                         cdm->pos.position_type =
1904                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS;
1905
1906                         cdm->pos.cookie.bus = bus;
1907                         cdm->pos.generations[CAM_BUS_GENERATION]=
1908                                 bus_generation;
1909                         cdm->status = CAM_DEV_MATCH_MORE;
1910                         return(0);
1911                 }
1912                 j = cdm->num_matches;
1913                 cdm->num_matches++;
1914                 cdm->matches[j].type = DEV_MATCH_BUS;
1915                 cdm->matches[j].result.bus_result.path_id = bus->path_id;
1916                 cdm->matches[j].result.bus_result.bus_id = bus->sim->bus_id;
1917                 cdm->matches[j].result.bus_result.unit_number =
1918                         bus->sim->unit_number;
1919                 strncpy(cdm->matches[j].result.bus_result.dev_name,
1920                         bus->sim->sim_name, DEV_IDLEN);
1921         }
1922
1923         /*
1924          * If the user is only interested in busses, there's no
1925          * reason to descend to the next level in the tree.
1926          */
1927         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
1928                 return(1);
1929
1930         /*
1931          * If there is a target generation recorded, check it to
1932          * make sure the target list hasn't changed.
1933          */
1934         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1935          && (bus == cdm->pos.cookie.bus)
1936          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1937          && (cdm->pos.generations[CAM_TARGET_GENERATION] != 0)
1938          && (cdm->pos.generations[CAM_TARGET_GENERATION] !=
1939              bus->generation)) {
1940                 cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1941                 return(0);
1942         }
1943
1944         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1945          && (cdm->pos.cookie.bus == bus)
1946          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1947          && (cdm->pos.cookie.target != NULL))
1948                 return(xpttargettraverse(bus,
1949                                         (struct cam_et *)cdm->pos.cookie.target,
1950                                          xptedttargetfunc, arg));
1951         else
1952                 return(xpttargettraverse(bus, NULL, xptedttargetfunc, arg));
1953 }
1954
1955 static int
1956 xptedttargetfunc(struct cam_et *target, void *arg)
1957 {
1958         struct ccb_dev_match *cdm;
1959
1960         cdm = (struct ccb_dev_match *)arg;
1961
1962         /*
1963          * If there is a device list generation recorded, check it to
1964          * make sure the device list hasn't changed.
1965          */
1966         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1967          && (cdm->pos.cookie.bus == target->bus)
1968          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1969          && (cdm->pos.cookie.target == target)
1970          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1971          && (cdm->pos.generations[CAM_DEV_GENERATION] != 0)
1972          && (cdm->pos.generations[CAM_DEV_GENERATION] !=
1973              target->generation)) {
1974                 cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
1975                 return(0);
1976         }
1977
1978         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
1979          && (cdm->pos.cookie.bus == target->bus)
1980          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
1981          && (cdm->pos.cookie.target == target)
1982          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
1983          && (cdm->pos.cookie.device != NULL))
1984                 return(xptdevicetraverse(target,
1985                                         (struct cam_ed *)cdm->pos.cookie.device,
1986                                          xptedtdevicefunc, arg));
1987         else
1988                 return(xptdevicetraverse(target, NULL, xptedtdevicefunc, arg));
1989 }
1990
1991 static int
1992 xptedtdevicefunc(struct cam_ed *device, void *arg)
1993 {
1994
1995         struct ccb_dev_match *cdm;
1996         dev_match_ret retval;
1997
1998         cdm = (struct ccb_dev_match *)arg;
1999
2000         /*
2001          * If our position is for something deeper in the tree, that means
2002          * that we've already seen this node.  So, we keep going down.
2003          */
2004         if ((cdm->pos.position_type & CAM_DEV_POS_DEVICE)
2005          && (cdm->pos.cookie.device == device)
2006          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
2007          && (cdm->pos.cookie.periph != NULL))
2008                 retval = DM_RET_DESCEND;
2009         else
2010                 retval = xptdevicematch(cdm->patterns, cdm->num_patterns,
2011                                         device);
2012
2013         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
2014                 cdm->status = CAM_DEV_MATCH_ERROR;
2015                 return(0);
2016         }
2017
2018         /*
2019          * If the copy flag is set, copy this device out.
2020          */
2021         if (retval & DM_RET_COPY) {
2022                 int spaceleft, j;
2023
2024                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
2025                         sizeof(struct dev_match_result));
2026
2027                 /*
2028                  * If we don't have enough space to put in another
2029                  * match result, save our position and tell the
2030                  * user there are more devices to check.
2031                  */
2032                 if (spaceleft < sizeof(struct dev_match_result)) {
2033                         bzero(&cdm->pos, sizeof(cdm->pos));
2034                         cdm->pos.position_type =
2035                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
2036                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE;
2037
2038                         cdm->pos.cookie.bus = device->target->bus;
2039                         cdm->pos.generations[CAM_BUS_GENERATION]=
2040                                 bus_generation;
2041                         cdm->pos.cookie.target = device->target;
2042                         cdm->pos.generations[CAM_TARGET_GENERATION] =
2043                                 device->target->bus->generation;
2044                         cdm->pos.cookie.device = device;
2045                         cdm->pos.generations[CAM_DEV_GENERATION] =
2046                                 device->target->generation;
2047                         cdm->status = CAM_DEV_MATCH_MORE;
2048                         return(0);
2049                 }
2050                 j = cdm->num_matches;
2051                 cdm->num_matches++;
2052                 cdm->matches[j].type = DEV_MATCH_DEVICE;
2053                 cdm->matches[j].result.device_result.path_id =
2054                         device->target->bus->path_id;
2055                 cdm->matches[j].result.device_result.target_id =
2056                         device->target->target_id;
2057                 cdm->matches[j].result.device_result.target_lun =
2058                         device->lun_id;
2059                 bcopy(&device->inq_data,
2060                       &cdm->matches[j].result.device_result.inq_data,
2061                       sizeof(struct scsi_inquiry_data));
2062
2063                 /* Let the user know whether this device is unconfigured */
2064                 if (device->flags & CAM_DEV_UNCONFIGURED)
2065                         cdm->matches[j].result.device_result.flags =
2066                                 DEV_RESULT_UNCONFIGURED;
2067                 else
2068                         cdm->matches[j].result.device_result.flags =
2069                                 DEV_RESULT_NOFLAG;
2070         }
2071
2072         /*
2073          * If the user isn't interested in peripherals, don't descend
2074          * the tree any further.
2075          */
2076         if ((retval & DM_RET_ACTION_MASK) == DM_RET_STOP)
2077                 return(1);
2078
2079         /*
2080          * If there is a peripheral list generation recorded, make sure
2081          * it hasn't changed.
2082          */
2083         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
2084          && (device->target->bus == cdm->pos.cookie.bus)
2085          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
2086          && (device->target == cdm->pos.cookie.target)
2087          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
2088          && (device == cdm->pos.cookie.device)
2089          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
2090          && (cdm->pos.generations[CAM_PERIPH_GENERATION] != 0)
2091          && (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
2092              device->generation)){
2093                 cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
2094                 return(0);
2095         }
2096
2097         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
2098          && (cdm->pos.cookie.bus == device->target->bus)
2099          && (cdm->pos.position_type & CAM_DEV_POS_TARGET)
2100          && (cdm->pos.cookie.target == device->target)
2101          && (cdm->pos.position_type & CAM_DEV_POS_DEVICE)
2102          && (cdm->pos.cookie.device == device)
2103          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
2104          && (cdm->pos.cookie.periph != NULL))
2105                 return(xptperiphtraverse(device,
2106                                 (struct cam_periph *)cdm->pos.cookie.periph,
2107                                 xptedtperiphfunc, arg));
2108         else
2109                 return(xptperiphtraverse(device, NULL, xptedtperiphfunc, arg));
2110 }
2111
2112 static int
2113 xptedtperiphfunc(struct cam_periph *periph, void *arg)
2114 {
2115         struct ccb_dev_match *cdm;
2116         dev_match_ret retval;
2117
2118         cdm = (struct ccb_dev_match *)arg;
2119
2120         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
2121
2122         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
2123                 cdm->status = CAM_DEV_MATCH_ERROR;
2124                 return(0);
2125         }
2126
2127         /*
2128          * If the copy flag is set, copy this peripheral out.
2129          */
2130         if (retval & DM_RET_COPY) {
2131                 int spaceleft, j;
2132
2133                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
2134                         sizeof(struct dev_match_result));
2135
2136                 /*
2137                  * If we don't have enough space to put in another
2138                  * match result, save our position and tell the
2139                  * user there are more devices to check.
2140                  */
2141                 if (spaceleft < sizeof(struct dev_match_result)) {
2142                         bzero(&cdm->pos, sizeof(cdm->pos));
2143                         cdm->pos.position_type =
2144                                 CAM_DEV_POS_EDT | CAM_DEV_POS_BUS |
2145                                 CAM_DEV_POS_TARGET | CAM_DEV_POS_DEVICE |
2146                                 CAM_DEV_POS_PERIPH;
2147
2148                         cdm->pos.cookie.bus = periph->path->bus;
2149                         cdm->pos.generations[CAM_BUS_GENERATION]=
2150                                 bus_generation;
2151                         cdm->pos.cookie.target = periph->path->target;
2152                         cdm->pos.generations[CAM_TARGET_GENERATION] =
2153                                 periph->path->bus->generation;
2154                         cdm->pos.cookie.device = periph->path->device;
2155                         cdm->pos.generations[CAM_DEV_GENERATION] =
2156                                 periph->path->target->generation;
2157                         cdm->pos.cookie.periph = periph;
2158                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
2159                                 periph->path->device->generation;
2160                         cdm->status = CAM_DEV_MATCH_MORE;
2161                         return(0);
2162                 }
2163
2164                 j = cdm->num_matches;
2165                 cdm->num_matches++;
2166                 cdm->matches[j].type = DEV_MATCH_PERIPH;
2167                 cdm->matches[j].result.periph_result.path_id =
2168                         periph->path->bus->path_id;
2169                 cdm->matches[j].result.periph_result.target_id =
2170                         periph->path->target->target_id;
2171                 cdm->matches[j].result.periph_result.target_lun =
2172                         periph->path->device->lun_id;
2173                 cdm->matches[j].result.periph_result.unit_number =
2174                         periph->unit_number;
2175                 strncpy(cdm->matches[j].result.periph_result.periph_name,
2176                         periph->periph_name, DEV_IDLEN);
2177         }
2178
2179         return(1);
2180 }
2181
2182 static int
2183 xptedtmatch(struct ccb_dev_match *cdm)
2184 {
2185         int ret;
2186
2187         cdm->num_matches = 0;
2188
2189         /*
2190          * Check the bus list generation.  If it has changed, the user
2191          * needs to reset everything and start over.
2192          */
2193         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
2194          && (cdm->pos.generations[CAM_BUS_GENERATION] != 0)
2195          && (cdm->pos.generations[CAM_BUS_GENERATION] != bus_generation)) {
2196                 cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
2197                 return(0);
2198         }
2199
2200         if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
2201          && (cdm->pos.cookie.bus != NULL))
2202                 ret = xptbustraverse((struct cam_eb *)cdm->pos.cookie.bus,
2203                                      xptedtbusfunc, cdm);
2204         else
2205                 ret = xptbustraverse(NULL, xptedtbusfunc, cdm);
2206
2207         /*
2208          * If we get back 0, that means that we had to stop before fully
2209          * traversing the EDT.  It also means that one of the subroutines
2210          * has set the status field to the proper value.  If we get back 1,
2211          * we've fully traversed the EDT and copied out any matching entries.
2212          */
2213         if (ret == 1)
2214                 cdm->status = CAM_DEV_MATCH_LAST;
2215
2216         return(ret);
2217 }
2218
2219 static int
2220 xptplistpdrvfunc(struct periph_driver **pdrv, void *arg)
2221 {
2222         struct ccb_dev_match *cdm;
2223
2224         cdm = (struct ccb_dev_match *)arg;
2225
2226         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2227          && (cdm->pos.cookie.pdrv == pdrv)
2228          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
2229          && (cdm->pos.generations[CAM_PERIPH_GENERATION] != 0)
2230          && (cdm->pos.generations[CAM_PERIPH_GENERATION] !=
2231              (*pdrv)->generation)) {
2232                 cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
2233                 return(0);
2234         }
2235
2236         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2237          && (cdm->pos.cookie.pdrv == pdrv)
2238          && (cdm->pos.position_type & CAM_DEV_POS_PERIPH)
2239          && (cdm->pos.cookie.periph != NULL))
2240                 return(xptpdperiphtraverse(pdrv,
2241                                 (struct cam_periph *)cdm->pos.cookie.periph,
2242                                 xptplistperiphfunc, arg));
2243         else
2244                 return(xptpdperiphtraverse(pdrv, NULL,xptplistperiphfunc, arg));
2245 }
2246
2247 static int
2248 xptplistperiphfunc(struct cam_periph *periph, void *arg)
2249 {
2250         struct ccb_dev_match *cdm;
2251         dev_match_ret retval;
2252
2253         cdm = (struct ccb_dev_match *)arg;
2254
2255         retval = xptperiphmatch(cdm->patterns, cdm->num_patterns, periph);
2256
2257         if ((retval & DM_RET_ACTION_MASK) == DM_RET_ERROR) {
2258                 cdm->status = CAM_DEV_MATCH_ERROR;
2259                 return(0);
2260         }
2261
2262         /*
2263          * If the copy flag is set, copy this peripheral out.
2264          */
2265         if (retval & DM_RET_COPY) {
2266                 int spaceleft, j;
2267
2268                 spaceleft = cdm->match_buf_len - (cdm->num_matches *
2269                         sizeof(struct dev_match_result));
2270
2271                 /*
2272                  * If we don't have enough space to put in another
2273                  * match result, save our position and tell the
2274                  * user there are more devices to check.
2275                  */
2276                 if (spaceleft < sizeof(struct dev_match_result)) {
2277                         struct periph_driver **pdrv;
2278
2279                         pdrv = NULL;
2280                         bzero(&cdm->pos, sizeof(cdm->pos));
2281                         cdm->pos.position_type =
2282                                 CAM_DEV_POS_PDRV | CAM_DEV_POS_PDPTR |
2283                                 CAM_DEV_POS_PERIPH;
2284
2285                         /*
2286                          * This may look a bit non-sensical, but it is
2287                          * actually quite logical.  There are very few
2288                          * peripheral drivers, and bloating every peripheral
2289                          * structure with a pointer back to its parent
2290                          * peripheral driver linker set entry would cost
2291                          * more in the long run than doing this quick lookup.
2292                          */
2293                         for (pdrv =
2294                              (struct periph_driver **)periphdriver_set.ls_items;
2295                              *pdrv != NULL; pdrv++) {
2296                                 if (strcmp((*pdrv)->driver_name,
2297                                     periph->periph_name) == 0)
2298                                         break;
2299                         }
2300
2301                         if (pdrv == NULL) {
2302                                 cdm->status = CAM_DEV_MATCH_ERROR;
2303                                 return(0);
2304                         }
2305
2306                         cdm->pos.cookie.pdrv = pdrv;
2307                         /*
2308                          * The periph generation slot does double duty, as
2309                          * does the periph pointer slot.  They are used for
2310                          * both edt and pdrv lookups and positioning.
2311                          */
2312                         cdm->pos.cookie.periph = periph;
2313                         cdm->pos.generations[CAM_PERIPH_GENERATION] =
2314                                 (*pdrv)->generation;
2315                         cdm->status = CAM_DEV_MATCH_MORE;
2316                         return(0);
2317                 }
2318
2319                 j = cdm->num_matches;
2320                 cdm->num_matches++;
2321                 cdm->matches[j].type = DEV_MATCH_PERIPH;
2322                 cdm->matches[j].result.periph_result.path_id =
2323                         periph->path->bus->path_id;
2324
2325                 /*
2326                  * The transport layer peripheral doesn't have a target or
2327                  * lun.
2328                  */
2329                 if (periph->path->target)
2330                         cdm->matches[j].result.periph_result.target_id =
2331                                 periph->path->target->target_id;
2332                 else
2333                         cdm->matches[j].result.periph_result.target_id = -1;
2334
2335                 if (periph->path->device)
2336                         cdm->matches[j].result.periph_result.target_lun =
2337                                 periph->path->device->lun_id;
2338                 else
2339                         cdm->matches[j].result.periph_result.target_lun = -1;
2340
2341                 cdm->matches[j].result.periph_result.unit_number =
2342                         periph->unit_number;
2343                 strncpy(cdm->matches[j].result.periph_result.periph_name,
2344                         periph->periph_name, DEV_IDLEN);
2345         }
2346
2347         return(1);
2348 }
2349
2350 static int
2351 xptperiphlistmatch(struct ccb_dev_match *cdm)
2352 {
2353         int ret;
2354
2355         cdm->num_matches = 0;
2356
2357         /*
2358          * At this point in the edt traversal function, we check the bus
2359          * list generation to make sure that no busses have been added or
2360          * removed since the user last sent a XPT_DEV_MATCH ccb through.
2361          * For the peripheral driver list traversal function, however, we
2362          * don't have to worry about new peripheral driver types coming or
2363          * going; they're in a linker set, and therefore can't change
2364          * without a recompile.
2365          */
2366
2367         if ((cdm->pos.position_type & CAM_DEV_POS_PDPTR)
2368          && (cdm->pos.cookie.pdrv != NULL))
2369                 ret = xptpdrvtraverse(
2370                                 (struct periph_driver **)cdm->pos.cookie.pdrv,
2371                                 xptplistpdrvfunc, cdm);
2372         else
2373                 ret = xptpdrvtraverse(NULL, xptplistpdrvfunc, cdm);
2374
2375         /*
2376          * If we get back 0, that means that we had to stop before fully
2377          * traversing the peripheral driver tree.  It also means that one of
2378          * the subroutines has set the status field to the proper value.  If
2379          * we get back 1, we've fully traversed the EDT and copied out any
2380          * matching entries.
2381          */
2382         if (ret == 1)
2383                 cdm->status = CAM_DEV_MATCH_LAST;
2384
2385         return(ret);
2386 }
2387
2388 static int
2389 xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
2390 {
2391         struct cam_eb *bus, *next_bus;
2392         int retval;
2393
2394         retval = 1;
2395
2396         for (bus = (start_bus ? start_bus : TAILQ_FIRST(&xpt_busses));
2397              bus != NULL;
2398              bus = next_bus) {
2399                 next_bus = TAILQ_NEXT(bus, links);
2400
2401                 retval = tr_func(bus, arg);
2402                 if (retval == 0)
2403                         return(retval);
2404         }
2405
2406         return(retval);
2407 }
2408
2409 static int
2410 xpttargettraverse(struct cam_eb *bus, struct cam_et *start_target,
2411                   xpt_targetfunc_t *tr_func, void *arg)
2412 {
2413         struct cam_et *target, *next_target;
2414         int retval;
2415
2416         retval = 1;
2417         for (target = (start_target ? start_target :
2418                        TAILQ_FIRST(&bus->et_entries));
2419              target != NULL; target = next_target) {
2420
2421                 next_target = TAILQ_NEXT(target, links);
2422
2423                 retval = tr_func(target, arg);
2424
2425                 if (retval == 0)
2426                         return(retval);
2427         }
2428
2429         return(retval);
2430 }
2431
2432 static int
2433 xptdevicetraverse(struct cam_et *target, struct cam_ed *start_device,
2434                   xpt_devicefunc_t *tr_func, void *arg)
2435 {
2436         struct cam_ed *device, *next_device;
2437         int retval;
2438
2439         retval = 1;
2440         for (device = (start_device ? start_device :
2441                        TAILQ_FIRST(&target->ed_entries));
2442              device != NULL;
2443              device = next_device) {
2444
2445                 next_device = TAILQ_NEXT(device, links);
2446
2447                 retval = tr_func(device, arg);
2448
2449                 if (retval == 0)
2450                         return(retval);
2451         }
2452
2453         return(retval);
2454 }
2455
2456 static int
2457 xptperiphtraverse(struct cam_ed *device, struct cam_periph *start_periph,
2458                   xpt_periphfunc_t *tr_func, void *arg)
2459 {
2460         struct cam_periph *periph, *next_periph;
2461         int retval;
2462
2463         retval = 1;
2464
2465         for (periph = (start_periph ? start_periph :
2466                        SLIST_FIRST(&device->periphs));
2467              periph != NULL;
2468              periph = next_periph) {
2469
2470                 next_periph = SLIST_NEXT(periph, periph_links);
2471
2472                 retval = tr_func(periph, arg);
2473                 if (retval == 0)
2474                         return(retval);
2475         }
2476
2477         return(retval);
2478 }
2479
2480 static int
2481 xptpdrvtraverse(struct periph_driver **start_pdrv,
2482                 xpt_pdrvfunc_t *tr_func, void *arg)
2483 {
2484         struct periph_driver **pdrv;
2485         int retval;
2486
2487         retval = 1;
2488
2489         /*
2490          * We don't traverse the peripheral driver list like we do the
2491          * other lists, because it is a linker set, and therefore cannot be
2492          * changed during runtime.  If the peripheral driver list is ever
2493          * re-done to be something other than a linker set (i.e. it can
2494          * change while the system is running), the list traversal should
2495          * be modified to work like the other traversal functions.
2496          */
2497         for (pdrv = (start_pdrv ? start_pdrv :
2498              (struct periph_driver **)periphdriver_set.ls_items);
2499              *pdrv != NULL; pdrv++) {
2500                 retval = tr_func(pdrv, arg);
2501
2502                 if (retval == 0)
2503                         return(retval);
2504         }
2505
2506         return(retval);
2507 }
2508
2509 static int
2510 xptpdperiphtraverse(struct periph_driver **pdrv,
2511                     struct cam_periph *start_periph,
2512                     xpt_periphfunc_t *tr_func, void *arg)
2513 {
2514         struct cam_periph *periph, *next_periph;
2515         int retval;
2516
2517         retval = 1;
2518
2519         for (periph = (start_periph ? start_periph :
2520              TAILQ_FIRST(&(*pdrv)->units)); periph != NULL;
2521              periph = next_periph) {
2522
2523                 next_periph = TAILQ_NEXT(periph, unit_links);
2524
2525                 retval = tr_func(periph, arg);
2526                 if (retval == 0)
2527                         return(retval);
2528         }
2529         return(retval);
2530 }
2531
2532 static int
2533 xptdefbusfunc(struct cam_eb *bus, void *arg)
2534 {
2535         struct xpt_traverse_config *tr_config;
2536
2537         tr_config = (struct xpt_traverse_config *)arg;
2538
2539         if (tr_config->depth == XPT_DEPTH_BUS) {
2540                 xpt_busfunc_t *tr_func;
2541
2542                 tr_func = (xpt_busfunc_t *)tr_config->tr_func;
2543
2544                 return(tr_func(bus, tr_config->tr_arg));
2545         } else
2546                 return(xpttargettraverse(bus, NULL, xptdeftargetfunc, arg));
2547 }
2548
2549 static int
2550 xptdeftargetfunc(struct cam_et *target, void *arg)
2551 {
2552         struct xpt_traverse_config *tr_config;
2553
2554         tr_config = (struct xpt_traverse_config *)arg;
2555
2556         if (tr_config->depth == XPT_DEPTH_TARGET) {
2557                 xpt_targetfunc_t *tr_func;
2558
2559                 tr_func = (xpt_targetfunc_t *)tr_config->tr_func;
2560
2561                 return(tr_func(target, tr_config->tr_arg));
2562         } else
2563                 return(xptdevicetraverse(target, NULL, xptdefdevicefunc, arg));
2564 }
2565
2566 static int
2567 xptdefdevicefunc(struct cam_ed *device, void *arg)
2568 {
2569         struct xpt_traverse_config *tr_config;
2570
2571         tr_config = (struct xpt_traverse_config *)arg;
2572
2573         if (tr_config->depth == XPT_DEPTH_DEVICE) {
2574                 xpt_devicefunc_t *tr_func;
2575
2576                 tr_func = (xpt_devicefunc_t *)tr_config->tr_func;
2577
2578                 return(tr_func(device, tr_config->tr_arg));
2579         } else
2580                 return(xptperiphtraverse(device, NULL, xptdefperiphfunc, arg));
2581 }
2582
2583 static int
2584 xptdefperiphfunc(struct cam_periph *periph, void *arg)
2585 {
2586         struct xpt_traverse_config *tr_config;
2587         xpt_periphfunc_t *tr_func;
2588
2589         tr_config = (struct xpt_traverse_config *)arg;
2590
2591         tr_func = (xpt_periphfunc_t *)tr_config->tr_func;
2592
2593         /*
2594          * Unlike the other default functions, we don't check for depth
2595          * here.  The peripheral driver level is the last level in the EDT,
2596          * so if we're here, we should execute the function in question.
2597          */
2598         return(tr_func(periph, tr_config->tr_arg));
2599 }
2600
2601 /*
2602  * Execute the given function for every bus in the EDT.
2603  */
2604 static int
2605 xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
2606 {
2607         struct xpt_traverse_config tr_config;
2608
2609         tr_config.depth = XPT_DEPTH_BUS;
2610         tr_config.tr_func = tr_func;
2611         tr_config.tr_arg = arg;
2612
2613         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2614 }
2615
2616 #ifdef notusedyet
2617 /*
2618  * Execute the given function for every target in the EDT.
2619  */
2620 static int
2621 xpt_for_all_targets(xpt_targetfunc_t *tr_func, void *arg)
2622 {
2623         struct xpt_traverse_config tr_config;
2624
2625         tr_config.depth = XPT_DEPTH_TARGET;
2626         tr_config.tr_func = tr_func;
2627         tr_config.tr_arg = arg;
2628
2629         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2630 }
2631 #endif /* notusedyet */
2632
2633 /*
2634  * Execute the given function for every device in the EDT.
2635  */
2636 static int
2637 xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
2638 {
2639         struct xpt_traverse_config tr_config;
2640
2641         tr_config.depth = XPT_DEPTH_DEVICE;
2642         tr_config.tr_func = tr_func;
2643         tr_config.tr_arg = arg;
2644
2645         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2646 }
2647
2648 #ifdef notusedyet
2649 /*
2650  * Execute the given function for every peripheral in the EDT.
2651  */
2652 static int
2653 xpt_for_all_periphs(xpt_periphfunc_t *tr_func, void *arg)
2654 {
2655         struct xpt_traverse_config tr_config;
2656
2657         tr_config.depth = XPT_DEPTH_PERIPH;
2658         tr_config.tr_func = tr_func;
2659         tr_config.tr_arg = arg;
2660
2661         return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
2662 }
2663 #endif /* notusedyet */
2664
2665 static int
2666 xptsetasyncfunc(struct cam_ed *device, void *arg)
2667 {
2668         struct cam_path path;
2669         struct ccb_getdev cgd;
2670         struct async_node *cur_entry;
2671
2672         cur_entry = (struct async_node *)arg;
2673
2674         /*
2675          * Don't report unconfigured devices (Wildcard devs,
2676          * devices only for target mode, device instances
2677          * that have been invalidated but are waiting for
2678          * their last reference count to be released).
2679          */
2680         if ((device->flags & CAM_DEV_UNCONFIGURED) != 0)
2681                 return (1);
2682
2683         xpt_compile_path(&path,
2684                          NULL,
2685                          device->target->bus->path_id,
2686                          device->target->target_id,
2687                          device->lun_id);
2688         xpt_setup_ccb(&cgd.ccb_h, &path, /*priority*/1);
2689         cgd.ccb_h.func_code = XPT_GDEV_TYPE;
2690         xpt_action((union ccb *)&cgd);
2691         cur_entry->callback(cur_entry->callback_arg,
2692                             AC_FOUND_DEVICE,
2693                             &path, &cgd);
2694         xpt_release_path(&path);
2695
2696         return(1);
2697 }
2698
2699 static int
2700 xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
2701 {
2702         struct cam_path path;
2703         struct ccb_pathinq cpi;
2704         struct async_node *cur_entry;
2705
2706         cur_entry = (struct async_node *)arg;
2707
2708         xpt_compile_path(&path, /*periph*/NULL,
2709                          bus->sim->path_id,
2710                          CAM_TARGET_WILDCARD,
2711                          CAM_LUN_WILDCARD);
2712         xpt_setup_ccb(&cpi.ccb_h, &path, /*priority*/1);
2713         cpi.ccb_h.func_code = XPT_PATH_INQ;
2714         xpt_action((union ccb *)&cpi);
2715         cur_entry->callback(cur_entry->callback_arg,
2716                             AC_PATH_REGISTERED,
2717                             &path, &cpi);
2718         xpt_release_path(&path);
2719
2720         return(1);
2721 }
2722
2723 void
2724 xpt_action(union ccb *start_ccb)
2725 {
2726         int iopl;
2727
2728         CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
2729
2730         start_ccb->ccb_h.status = CAM_REQ_INPROG;
2731
2732         iopl = splsoftcam();
2733         switch (start_ccb->ccb_h.func_code) {
2734         case XPT_SCSI_IO:
2735         {
2736 #ifdef CAMDEBUG
2737                 char cdb_str[(SCSI_MAX_CDBLEN * 3) + 1];
2738                 struct cam_path *path;
2739
2740                 path = start_ccb->ccb_h.path;
2741 #endif
2742
2743                 /*
2744                  * For the sake of compatibility with SCSI-1
2745                  * devices that may not understand the identify
2746                  * message, we include lun information in the
2747                  * second byte of all commands.  SCSI-1 specifies
2748                  * that luns are a 3 bit value and reserves only 3
2749                  * bits for lun information in the CDB.  Later
2750                  * revisions of the SCSI spec allow for more than 8
2751                  * luns, but have deprecated lun information in the
2752                  * CDB.  So, if the lun won't fit, we must omit.
2753                  *
2754                  * Also be aware that during initial probing for devices,
2755                  * the inquiry information is unknown but initialized to 0.
2756                  * This means that this code will be exercised while probing
2757                  * devices with an ANSI revision greater than 2.
2758                  */
2759                 if (SID_ANSI_REV(&start_ccb->ccb_h.path->device->inq_data) <= 2
2760                  && start_ccb->ccb_h.target_lun < 8
2761                  && (start_ccb->ccb_h.flags & CAM_CDB_POINTER) == 0) {
2762
2763                         start_ccb->csio.cdb_io.cdb_bytes[1] |=
2764                             start_ccb->ccb_h.target_lun << 5;
2765                 }
2766                 start_ccb->csio.scsi_status = SCSI_STATUS_OK;
2767                 CAM_DEBUG(path, CAM_DEBUG_CDB,("%s. CDB: %s\n",
2768                           scsi_op_desc(start_ccb->csio.cdb_io.cdb_bytes[0],
2769                                        &path->device->inq_data),
2770                           scsi_cdb_string(start_ccb->csio.cdb_io.cdb_bytes,
2771                                           cdb_str, sizeof(cdb_str))));
2772                 /* FALLTHROUGH */
2773         }
2774         case XPT_TARGET_IO:
2775         case XPT_CONT_TARGET_IO:
2776                 start_ccb->csio.sense_resid = 0;
2777                 start_ccb->csio.resid = 0;
2778                 /* FALLTHROUGH */
2779         case XPT_RESET_DEV:
2780         case XPT_ENG_EXEC:
2781         {
2782                 struct cam_path *path;
2783                 int s;
2784                 int runq;
2785
2786                 path = start_ccb->ccb_h.path;
2787                 s = splsoftcam();
2788
2789                 cam_ccbq_insert_ccb(&path->device->ccbq, start_ccb);
2790                 if (path->device->qfrozen_cnt == 0)
2791                         runq = xpt_schedule_dev_sendq(path->bus, path->device);
2792                 else
2793                         runq = 0;
2794                 splx(s);
2795                 if (runq != 0)
2796                         xpt_run_dev_sendq(path->bus);
2797                 break;
2798         }
2799         case XPT_SET_TRAN_SETTINGS:
2800         {
2801                 xpt_set_transfer_settings(&start_ccb->cts,
2802                                           start_ccb->ccb_h.path->device,
2803                                           /*async_update*/FALSE);
2804                 break;
2805         }
2806         case XPT_CALC_GEOMETRY:
2807         {
2808                 struct cam_sim *sim;
2809
2810                 /* Filter out garbage */
2811                 if (start_ccb->ccg.block_size == 0
2812                  || start_ccb->ccg.volume_size == 0) {
2813                         start_ccb->ccg.cylinders = 0;
2814                         start_ccb->ccg.heads = 0;
2815                         start_ccb->ccg.secs_per_track = 0;
2816                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2817                         break;
2818                 }
2819 #ifdef PC98
2820                 /*
2821                  * In a PC-98 system, geometry translation depens on
2822                  * the "real" device geometry obtained from mode page 4.
2823                  * SCSI geometry translation is performed in the
2824                  * initialization routine of the SCSI BIOS and the result
2825                  * stored in host memory.  If the translation is available
2826                  * in host memory, use it.  If not, rely on the default
2827                  * translation the device driver performs.
2828                  */
2829                 if (scsi_da_bios_params(&start_ccb->ccg) != 0) {
2830                         start_ccb->ccb_h.status = CAM_REQ_CMP;
2831                         break;
2832                 }
2833 #endif
2834                 sim = start_ccb->ccb_h.path->bus->sim;
2835                 (*(sim->sim_action))(sim, start_ccb);
2836                 break;
2837         }
2838         case XPT_ABORT:
2839         {
2840                 union ccb* abort_ccb;
2841                 int s;
2842
2843                 abort_ccb = start_ccb->cab.abort_ccb;
2844                 if (XPT_FC_IS_DEV_QUEUED(abort_ccb)) {
2845
2846                         if (abort_ccb->ccb_h.pinfo.index >= 0) {
2847                                 struct cam_ccbq *ccbq;
2848
2849                                 ccbq = &abort_ccb->ccb_h.path->device->ccbq;
2850                                 cam_ccbq_remove_ccb(ccbq, abort_ccb);
2851                                 abort_ccb->ccb_h.status =
2852                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2853                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2854                                 s = splcam();
2855                                 xpt_done(abort_ccb);
2856                                 splx(s);
2857                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2858                                 break;
2859                         }
2860                         if (abort_ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX
2861                          && (abort_ccb->ccb_h.status & CAM_SIM_QUEUED) == 0) {
2862                                 /*
2863                                  * We've caught this ccb en route to
2864                                  * the SIM.  Flag it for abort and the
2865                                  * SIM will do so just before starting
2866                                  * real work on the CCB.
2867                                  */
2868                                 abort_ccb->ccb_h.status =
2869                                     CAM_REQ_ABORTED|CAM_DEV_QFRZN;
2870                                 xpt_freeze_devq(abort_ccb->ccb_h.path, 1);
2871                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2872                                 break;
2873                         }
2874                 }
2875                 if (XPT_FC_IS_QUEUED(abort_ccb)
2876                  && (abort_ccb->ccb_h.pinfo.index == CAM_DONEQ_INDEX)) {
2877                         /*
2878                          * It's already completed but waiting
2879                          * for our SWI to get to it.
2880                          */
2881                         start_ccb->ccb_h.status = CAM_UA_ABORT;
2882                         break;
2883                 }
2884                 /*
2885                  * If we weren't able to take care of the abort request
2886                  * in the XPT, pass the request down to the SIM for processing.
2887                  */
2888                 /* FALLTHROUGH */
2889         }
2890         case XPT_ACCEPT_TARGET_IO:
2891         case XPT_EN_LUN:
2892         case XPT_IMMED_NOTIFY:
2893         case XPT_NOTIFY_ACK:
2894         case XPT_GET_TRAN_SETTINGS:
2895         case XPT_RESET_BUS:
2896         {
2897                 struct cam_sim *sim;
2898
2899                 sim = start_ccb->ccb_h.path->bus->sim;
2900                 (*(sim->sim_action))(sim, start_ccb);
2901                 break;
2902         }
2903         case XPT_PATH_INQ:
2904         {
2905                 struct cam_sim *sim;
2906
2907                 sim = start_ccb->ccb_h.path->bus->sim;
2908                 (*(sim->sim_action))(sim, start_ccb);
2909                 break;
2910         }
2911         case XPT_PATH_STATS:
2912                 start_ccb->cpis.last_reset =
2913                         start_ccb->ccb_h.path->bus->last_reset;
2914                 start_ccb->ccb_h.status = CAM_REQ_CMP;
2915                 break;
2916         case XPT_GDEV_TYPE:
2917         {
2918                 struct cam_ed *dev;
2919                 int s;
2920
2921                 dev = start_ccb->ccb_h.path->device;
2922                 s = splcam();
2923                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2924                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2925                 } else {
2926                         struct ccb_getdev *cgd;
2927                         struct cam_eb *bus;
2928                         struct cam_et *tar;
2929
2930                         cgd = &start_ccb->cgd;
2931                         bus = cgd->ccb_h.path->bus;
2932                         tar = cgd->ccb_h.path->target;
2933                         cgd->inq_data = dev->inq_data;
2934                         cgd->ccb_h.status = CAM_REQ_CMP;
2935                         cgd->serial_num_len = dev->serial_num_len;
2936                         if ((dev->serial_num_len > 0)
2937                          && (dev->serial_num != NULL))
2938                                 bcopy(dev->serial_num, cgd->serial_num,
2939                                       dev->serial_num_len);
2940                 }
2941                 splx(s);
2942                 break;
2943         }
2944         case XPT_GDEV_STATS:
2945         {
2946                 struct cam_ed *dev;
2947                 int s;
2948
2949                 dev = start_ccb->ccb_h.path->device;
2950                 s = splcam();
2951                 if ((dev->flags & CAM_DEV_UNCONFIGURED) != 0) {
2952                         start_ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2953                 } else {
2954                         struct ccb_getdevstats *cgds;
2955                         struct cam_eb *bus;
2956                         struct cam_et *tar;
2957
2958                         cgds = &start_ccb->cgds;
2959                         bus = cgds->ccb_h.path->bus;
2960                         tar = cgds->ccb_h.path->target;
2961                         cgds->dev_openings = dev->ccbq.dev_openings;
2962                         cgds->dev_active = dev->ccbq.dev_active;
2963                         cgds->devq_openings = dev->ccbq.devq_openings;
2964                         cgds->devq_queued = dev->ccbq.queue.entries;
2965                         cgds->held = dev->ccbq.held;
2966                         cgds->last_reset = tar->last_reset;
2967                         cgds->maxtags = dev->quirk->maxtags;
2968                         cgds->mintags = dev->quirk->mintags;
2969                         if (timevalcmp(&tar->last_reset, &bus->last_reset, <))
2970                                 cgds->last_reset = bus->last_reset;
2971                         cgds->ccb_h.status = CAM_REQ_CMP;
2972                 }
2973                 splx(s);
2974                 break;
2975         }
2976         case XPT_GDEVLIST:
2977         {
2978                 struct cam_periph       *nperiph;
2979                 struct periph_list      *periph_head;
2980                 struct ccb_getdevlist   *cgdl;
2981                 int                     i;
2982                 int                     s;
2983                 struct cam_ed           *device;
2984                 int                     found;
2985
2986
2987                 found = 0;
2988
2989                 /*
2990                  * Don't want anyone mucking with our data.
2991                  */
2992                 s = splcam();
2993                 device = start_ccb->ccb_h.path->device;
2994                 periph_head = &device->periphs;
2995                 cgdl = &start_ccb->cgdl;
2996
2997                 /*
2998                  * Check and see if the list has changed since the user
2999                  * last requested a list member.  If so, tell them that the
3000                  * list has changed, and therefore they need to start over
3001                  * from the beginning.
3002                  */
3003                 if ((cgdl->index != 0) &&
3004                     (cgdl->generation != device->generation)) {
3005                         cgdl->status = CAM_GDEVLIST_LIST_CHANGED;
3006                         splx(s);
3007                         break;
3008                 }
3009
3010                 /*
3011                  * Traverse the list of peripherals and attempt to find
3012                  * the requested peripheral.
3013                  */
3014                 for (nperiph = periph_head->slh_first, i = 0;
3015                      (nperiph != NULL) && (i <= cgdl->index);
3016                      nperiph = nperiph->periph_links.sle_next, i++) {
3017                         if (i == cgdl->index) {
3018                                 strncpy(cgdl->periph_name,
3019                                         nperiph->periph_name,
3020                                         DEV_IDLEN);
3021                                 cgdl->unit_number = nperiph->unit_number;
3022                                 found = 1;
3023                         }
3024                 }
3025                 if (found == 0) {
3026                         cgdl->status = CAM_GDEVLIST_ERROR;
3027                         splx(s);
3028                         break;
3029                 }
3030
3031                 if (nperiph == NULL)
3032                         cgdl->status = CAM_GDEVLIST_LAST_DEVICE;
3033                 else
3034                         cgdl->status = CAM_GDEVLIST_MORE_DEVS;
3035
3036                 cgdl->index++;
3037                 cgdl->generation = device->generation;
3038
3039                 splx(s);
3040                 cgdl->ccb_h.status = CAM_REQ_CMP;
3041                 break;
3042         }
3043         case XPT_DEV_MATCH:
3044         {
3045                 int s;
3046                 dev_pos_type position_type;
3047                 struct ccb_dev_match *cdm;
3048                 int ret;
3049
3050                 cdm = &start_ccb->cdm;
3051
3052                 /*
3053                  * Prevent EDT changes while we traverse it.
3054                  */
3055                 s = splcam();
3056                 /*
3057                  * There are two ways of getting at information in the EDT.
3058                  * The first way is via the primary EDT tree.  It starts
3059                  * with a list of busses, then a list of targets on a bus,
3060                  * then devices/luns on a target, and then peripherals on a
3061                  * device/lun.  The "other" way is by the peripheral driver
3062                  * lists.  The peripheral driver lists are organized by
3063                  * peripheral driver.  (obviously)  So it makes sense to
3064                  * use the peripheral driver list if the user is looking
3065                  * for something like "da1", or all "da" devices.  If the
3066                  * user is looking for something on a particular bus/target
3067                  * or lun, it's generally better to go through the EDT tree.
3068                  */
3069
3070                 if (cdm->pos.position_type != CAM_DEV_POS_NONE)
3071                         position_type = cdm->pos.position_type;
3072                 else {
3073                         int i;
3074
3075                         position_type = CAM_DEV_POS_NONE;
3076
3077                         for (i = 0; i < cdm->num_patterns; i++) {
3078                                 if ((cdm->patterns[i].type == DEV_MATCH_BUS)
3079                                  ||(cdm->patterns[i].type == DEV_MATCH_DEVICE)){
3080                                         position_type = CAM_DEV_POS_EDT;
3081                                         break;
3082                                 }
3083                         }
3084
3085                         if (cdm->num_patterns == 0)
3086                                 position_type = CAM_DEV_POS_EDT;
3087                         else if (position_type == CAM_DEV_POS_NONE)
3088                                 position_type = CAM_DEV_POS_PDRV;
3089                 }
3090
3091                 switch(position_type & CAM_DEV_POS_TYPEMASK) {
3092                 case CAM_DEV_POS_EDT:
3093                         ret = xptedtmatch(cdm);
3094                         break;
3095                 case CAM_DEV_POS_PDRV:
3096                         ret = xptperiphlistmatch(cdm);
3097                         break;
3098                 default:
3099                         cdm->status = CAM_DEV_MATCH_ERROR;
3100                         break;
3101                 }
3102
3103                 splx(s);
3104
3105                 if (cdm->status == CAM_DEV_MATCH_ERROR)
3106                         start_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
3107                 else
3108                         start_ccb->ccb_h.status = CAM_REQ_CMP;
3109
3110                 break;
3111         }
3112         case XPT_SASYNC_CB:
3113         {
3114                 struct ccb_setasync *csa;
3115                 struct async_node *cur_entry;
3116                 struct async_list *async_head;
3117                 u_int32_t added;
3118                 int s;
3119
3120                 csa = &start_ccb->csa;
3121                 added = csa->event_enable;
3122                 async_head = &csa->ccb_h.path->device->asyncs;
3123
3124                 /*
3125                  * If there is already an entry for us, simply
3126                  * update it.
3127                  */
3128                 s = splcam();
3129                 cur_entry = SLIST_FIRST(async_head);
3130                 while (cur_entry != NULL) {
3131                         if ((cur_entry->callback_arg == csa->callback_arg)
3132                          && (cur_entry->callback == csa->callback))
3133                                 break;
3134                         cur_entry = SLIST_NEXT(cur_entry, links);
3135                 }
3136
3137                 if (cur_entry != NULL) {
3138                         /*
3139                          * If the request has no flags set,
3140                          * remove the entry.
3141                          */
3142                         added &= ~cur_entry->event_enable;
3143                         if (csa->event_enable == 0) {
3144                                 SLIST_REMOVE(async_head, cur_entry,
3145                                              async_node, links);
3146                                 csa->ccb_h.path->device->refcount--;
3147                                 free(cur_entry, M_DEVBUF);
3148                         } else {
3149                                 cur_entry->event_enable = csa->event_enable;
3150                         }
3151                 } else {
3152                         cur_entry = malloc(sizeof(*cur_entry), M_DEVBUF,
3153                                            M_NOWAIT);
3154                         if (cur_entry == NULL) {
3155                                 splx(s);
3156                                 csa->ccb_h.status = CAM_RESRC_UNAVAIL;
3157                                 break;
3158                         }
3159                         cur_entry->event_enable = csa->event_enable;
3160                         cur_entry->callback_arg = csa->callback_arg;
3161                         cur_entry->callback = csa->callback;
3162                         SLIST_INSERT_HEAD(async_head, cur_entry, links);
3163                         csa->ccb_h.path->device->refcount++;
3164                 }
3165
3166                 if ((added & AC_FOUND_DEVICE) != 0) {
3167                         /*
3168                          * Get this peripheral up to date with all
3169                          * the currently existing devices.
3170                          */
3171                         xpt_for_all_devices(xptsetasyncfunc, cur_entry);
3172                 }
3173                 if ((added & AC_PATH_REGISTERED) != 0) {
3174                         /*
3175                          * Get this peripheral up to date with all
3176                          * the currently existing busses.
3177                          */
3178                         xpt_for_all_busses(xptsetasyncbusfunc, cur_entry);
3179                 }
3180                 splx(s);
3181                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3182                 break;
3183         }
3184         case XPT_REL_SIMQ:
3185         {
3186                 struct ccb_relsim *crs;
3187                 struct cam_ed *dev;
3188                 int s;
3189
3190                 crs = &start_ccb->crs;
3191                 dev = crs->ccb_h.path->device;
3192                 if (dev == NULL) {
3193
3194                         crs->ccb_h.status = CAM_DEV_NOT_THERE;
3195                         break;
3196                 }
3197
3198                 s = splcam();
3199
3200                 if ((crs->release_flags & RELSIM_ADJUST_OPENINGS) != 0) {
3201
3202                         if ((dev->inq_data.flags & SID_CmdQue) != 0) {
3203
3204                                 /* Don't ever go below one opening */
3205                                 if (crs->openings > 0) {
3206                                         xpt_dev_ccbq_resize(crs->ccb_h.path,
3207                                                             crs->openings);
3208
3209                                         if (bootverbose) {
3210                                                 xpt_print_path(crs->ccb_h.path);
3211                                                 printf("tagged openings "
3212                                                        "now %d\n",
3213                                                        crs->openings);
3214                                         }
3215                                 }
3216                         }
3217                 }
3218
3219                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_TIMEOUT) != 0) {
3220
3221                         if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
3222
3223                                 /*
3224                                  * Just extend the old timeout and decrement
3225                                  * the freeze count so that a single timeout
3226                                  * is sufficient for releasing the queue.
3227                                  */
3228                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3229                                 untimeout(xpt_release_devq_timeout,
3230                                           dev, dev->c_handle);
3231                         } else {
3232
3233                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3234                         }
3235
3236                         dev->c_handle =
3237                                 timeout(xpt_release_devq_timeout,
3238                                         dev,
3239                                         (crs->release_timeout * hz) / 1000);
3240
3241                         dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
3242
3243                 }
3244
3245                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_CMDCMPLT) != 0) {
3246
3247                         if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0) {
3248                                 /*
3249                                  * Decrement the freeze count so that a single
3250                                  * completion is still sufficient to unfreeze
3251                                  * the queue.
3252                                  */
3253                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3254                         } else {
3255
3256                                 dev->flags |= CAM_DEV_REL_ON_COMPLETE;
3257                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3258                         }
3259                 }
3260
3261                 if ((crs->release_flags & RELSIM_RELEASE_AFTER_QEMPTY) != 0) {
3262
3263                         if ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
3264                          || (dev->ccbq.dev_active == 0)) {
3265
3266                                 start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
3267                         } else {
3268
3269                                 dev->flags |= CAM_DEV_REL_ON_QUEUE_EMPTY;
3270                                 start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
3271                         }
3272                 }
3273                 splx(s);
3274
3275                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) == 0) {
3276
3277                         xpt_release_devq(crs->ccb_h.path, /*count*/1,
3278                                          /*run_queue*/TRUE);
3279                 }
3280                 start_ccb->crs.qfrozen_cnt = dev->qfrozen_cnt;
3281                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3282                 break;
3283         }
3284         case XPT_SCAN_BUS:
3285                 xpt_scan_bus(start_ccb->ccb_h.path->periph, start_ccb);
3286                 break;
3287         case XPT_SCAN_LUN:
3288                 xpt_scan_lun(start_ccb->ccb_h.path->periph,
3289                              start_ccb->ccb_h.path, start_ccb->crcn.flags,
3290                              start_ccb);
3291                 break;
3292         case XPT_DEBUG: {
3293 #ifdef CAMDEBUG
3294                 int s;
3295
3296                 s = splcam();
3297 #ifdef CAM_DEBUG_DELAY
3298                 cam_debug_delay = CAM_DEBUG_DELAY;
3299 #endif
3300                 cam_dflags = start_ccb->cdbg.flags;
3301                 if (cam_dpath != NULL) {
3302                         xpt_free_path(cam_dpath);
3303                         cam_dpath = NULL;
3304                 }
3305
3306                 if (cam_dflags != CAM_DEBUG_NONE) {
3307                         if (xpt_create_path(&cam_dpath, xpt_periph,
3308                                             start_ccb->ccb_h.path_id,
3309                                             start_ccb->ccb_h.target_id,
3310                                             start_ccb->ccb_h.target_lun) !=
3311                                             CAM_REQ_CMP) {
3312                                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3313                                 cam_dflags = CAM_DEBUG_NONE;
3314                         } else {
3315                                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3316                                 xpt_print_path(cam_dpath);
3317                                 printf("debugging flags now %x\n", cam_dflags);
3318                         }
3319                 } else {
3320                         cam_dpath = NULL;
3321                         start_ccb->ccb_h.status = CAM_REQ_CMP;
3322                 }
3323                 splx(s);
3324 #else /* !CAMDEBUG */
3325                 start_ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
3326 #endif /* CAMDEBUG */
3327                 break;
3328         }
3329         case XPT_NOOP:
3330                 if ((start_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0)
3331                         xpt_freeze_devq(start_ccb->ccb_h.path, 1);
3332                 start_ccb->ccb_h.status = CAM_REQ_CMP;
3333                 break;
3334         default:
3335         case XPT_SDEV_TYPE:
3336         case XPT_TERM_IO:
3337         case XPT_ENG_INQ:
3338                 /* XXX Implement */
3339                 start_ccb->ccb_h.status = CAM_PROVIDE_FAIL;
3340                 break;
3341         }
3342         splx(iopl);
3343 }
3344
3345 void
3346 xpt_polled_action(union ccb *start_ccb)
3347 {
3348         int       s;
3349         u_int32_t timeout;
3350         struct    cam_sim *sim;
3351         struct    cam_devq *devq;
3352         struct    cam_ed *dev;
3353
3354         timeout = start_ccb->ccb_h.timeout;
3355         sim = start_ccb->ccb_h.path->bus->sim;
3356         devq = sim->devq;
3357         dev = start_ccb->ccb_h.path->device;
3358
3359         s = splcam();
3360
3361         /*
3362          * Steal an opening so that no other queued requests
3363          * can get it before us while we simulate interrupts.
3364          */
3365         dev->ccbq.devq_openings--;
3366         dev->ccbq.dev_openings--;
3367
3368         while((devq->send_openings <= 0 || dev->ccbq.dev_openings < 0)
3369            && (--timeout > 0)) {
3370                 DELAY(1000);
3371                 (*(sim->sim_poll))(sim);
3372                 swi_camnet();
3373                 swi_cambio();
3374         }
3375
3376         dev->ccbq.devq_openings++;
3377         dev->ccbq.dev_openings++;
3378
3379         if (timeout != 0) {
3380                 xpt_action(start_ccb);
3381                 while(--timeout > 0) {
3382                         (*(sim->sim_poll))(sim);
3383                         swi_camnet();
3384                         swi_cambio();
3385                         if ((start_ccb->ccb_h.status  & CAM_STATUS_MASK)
3386                             != CAM_REQ_INPROG)
3387                                 break;
3388                         DELAY(1000);
3389                 }
3390                 if (timeout == 0) {
3391                         /*
3392                          * XXX Is it worth adding a sim_timeout entry
3393                          * point so we can attempt recovery?  If
3394                          * this is only used for dumps, I don't think
3395                          * it is.
3396                          */
3397                         start_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
3398                 }
3399         } else {
3400                 start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3401         }
3402         splx(s);
3403 }
3404
3405 /*
3406  * Schedule a peripheral driver to receive a ccb when it's
3407  * target device has space for more transactions.
3408  */
3409 void
3410 xpt_schedule(struct cam_periph *perph, u_int32_t new_priority)
3411 {
3412         struct cam_ed *device;
3413         int s;
3414         int runq;
3415
3416         CAM_DEBUG(perph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
3417         device = perph->path->device;
3418         s = splsoftcam();
3419         if (periph_is_queued(perph)) {
3420                 /* Simply reorder based on new priority */
3421                 CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE,
3422                           ("   change priority to %d\n", new_priority));
3423                 if (new_priority < perph->pinfo.priority) {
3424                         camq_change_priority(&device->drvq,
3425                                              perph->pinfo.index,
3426                                              new_priority);
3427                 }
3428                 runq = 0;
3429         } else {
3430                 /* New entry on the queue */
3431                 CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE,
3432                           ("   added periph to queue\n"));
3433                 perph->pinfo.priority = new_priority;
3434                 perph->pinfo.generation = ++device->drvq.generation;
3435                 camq_insert(&device->drvq, &perph->pinfo);
3436                 runq = xpt_schedule_dev_allocq(perph->path->bus, device);
3437         }
3438         splx(s);
3439         if (runq != 0) {
3440                 CAM_DEBUG(perph->path, CAM_DEBUG_SUBTRACE,
3441                           ("   calling xpt_run_devq\n"));
3442                 xpt_run_dev_allocq(perph->path->bus);
3443         }
3444 }
3445
3446
3447 /*
3448  * Schedule a device to run on a given queue.
3449  * If the device was inserted as a new entry on the queue,
3450  * return 1 meaning the device queue should be run. If we
3451  * were already queued, implying someone else has already
3452  * started the queue, return 0 so the caller doesn't attempt
3453  * to run the queue.  Must be run at either splsoftcam
3454  * (or splcam since that encompases splsoftcam).
3455  */
3456 static int
3457 xpt_schedule_dev(struct camq *queue, cam_pinfo *pinfo,
3458                  u_int32_t new_priority)
3459 {
3460         int retval;
3461         u_int32_t old_priority;
3462
3463         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_schedule_dev\n"));
3464
3465         old_priority = pinfo->priority;
3466
3467         /*
3468          * Are we already queued?
3469          */
3470         if (pinfo->index != CAM_UNQUEUED_INDEX) {
3471                 /* Simply reorder based on new priority */
3472                 if (new_priority < old_priority) {
3473                         camq_change_priority(queue, pinfo->index,
3474                                              new_priority);
3475                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3476                                         ("changed priority to %d\n",
3477                                          new_priority));
3478                 }
3479                 retval = 0;
3480         } else {
3481                 /* New entry on the queue */
3482                 if (new_priority < old_priority)
3483                         pinfo->priority = new_priority;
3484
3485                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3486                                 ("Inserting onto queue\n"));
3487                 pinfo->generation = ++queue->generation;
3488                 camq_insert(queue, pinfo);
3489                 retval = 1;
3490         }
3491         return (retval);
3492 }
3493
3494 static void
3495 xpt_run_dev_allocq(struct cam_eb *bus)
3496 {
3497         struct  cam_devq *devq;
3498         int     s;
3499
3500         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_dev_allocq\n"));
3501         devq = bus->sim->devq;
3502
3503         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3504                         ("   qfrozen_cnt == 0x%x, entries == %d, "
3505                          "openings == %d, active == %d\n",
3506                          devq->alloc_queue.qfrozen_cnt,
3507                          devq->alloc_queue.entries,
3508                          devq->alloc_openings,
3509                          devq->alloc_active));
3510
3511         s = splsoftcam();
3512         devq->alloc_queue.qfrozen_cnt++;
3513         while ((devq->alloc_queue.entries > 0)
3514             && (devq->alloc_openings > 0)
3515             && (devq->alloc_queue.qfrozen_cnt <= 1)) {
3516                 struct  cam_ed_qinfo *qinfo;
3517                 struct  cam_ed *device;
3518                 union   ccb *work_ccb;
3519                 struct  cam_periph *drv;
3520                 struct  camq *drvq;
3521
3522                 qinfo = (struct cam_ed_qinfo *)camq_remove(&devq->alloc_queue,
3523                                                            CAMQ_HEAD);
3524                 device = qinfo->device;
3525
3526                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3527                                 ("running device %p\n", device));
3528
3529                 drvq = &device->drvq;
3530
3531 #ifdef CAMDEBUG
3532                 if (drvq->entries <= 0) {
3533                         panic("xpt_run_dev_allocq: "
3534                               "Device on queue without any work to do");
3535                 }
3536 #endif
3537                 if ((work_ccb = xpt_get_ccb(device)) != NULL) {
3538                         devq->alloc_openings--;
3539                         devq->alloc_active++;
3540                         drv = (struct cam_periph*)camq_remove(drvq, CAMQ_HEAD);
3541                         splx(s);
3542                         xpt_setup_ccb(&work_ccb->ccb_h, drv->path,
3543                                       drv->pinfo.priority);
3544                         CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3545                                         ("calling periph start\n"));
3546                         drv->periph_start(drv, work_ccb);
3547                 } else {
3548                         /*
3549                          * Malloc failure in alloc_ccb
3550                          */
3551                         /*
3552                          * XXX add us to a list to be run from free_ccb
3553                          * if we don't have any ccbs active on this
3554                          * device queue otherwise we may never get run
3555                          * again.
3556                          */
3557                         break;
3558                 }
3559
3560                 /* Raise IPL for possible insertion and test at top of loop */
3561                 s = splsoftcam();
3562
3563                 if (drvq->entries > 0) {
3564                         /* We have more work.  Attempt to reschedule */
3565                         xpt_schedule_dev_allocq(bus, device);
3566                 }
3567         }
3568         devq->alloc_queue.qfrozen_cnt--;
3569         splx(s);
3570 }
3571
3572 static void
3573 xpt_run_dev_sendq(struct cam_eb *bus)
3574 {
3575         struct  cam_devq *devq;
3576         int     s;
3577
3578         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_run_dev_sendq\n"));
3579
3580         devq = bus->sim->devq;
3581
3582         s = splcam();
3583         devq->send_queue.qfrozen_cnt++;
3584         splx(s);
3585         s = splsoftcam();
3586         while ((devq->send_queue.entries > 0)
3587             && (devq->send_openings > 0)) {
3588                 struct  cam_ed_qinfo *qinfo;
3589                 struct  cam_ed *device;
3590                 union ccb *work_ccb;
3591                 struct  cam_sim *sim;
3592                 int     ospl;
3593
3594                 ospl = splcam();
3595                 if (devq->send_queue.qfrozen_cnt > 1) {
3596                         splx(ospl);
3597                         break;
3598                 }
3599
3600                 qinfo = (struct cam_ed_qinfo *)camq_remove(&devq->send_queue,
3601                                                            CAMQ_HEAD);
3602                 device = qinfo->device;
3603
3604                 /*
3605                  * If the device has been "frozen", don't attempt
3606                  * to run it.
3607                  */
3608                 if (device->qfrozen_cnt > 0) {
3609                         splx(ospl);
3610                         continue;
3611                 }
3612
3613                 CAM_DEBUG_PRINT(CAM_DEBUG_XPT,
3614                                 ("running device %p\n", device));
3615
3616                 work_ccb = cam_ccbq_peek_ccb(&device->ccbq, CAMQ_HEAD);
3617                 if (work_ccb == NULL) {
3618                         printf("device on run queue with no ccbs???");
3619                         splx(ospl);
3620                         continue;
3621                 }
3622
3623                 if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
3624
3625                         if (num_highpower <= 0) {
3626                                 /*
3627                                  * We got a high power command, but we
3628                                  * don't have any available slots.  Freeze
3629                                  * the device queue until we have a slot
3630                                  * available.
3631                                  */
3632                                 device->qfrozen_cnt++;
3633                                 STAILQ_INSERT_TAIL(&highpowerq,
3634                                                    &work_ccb->ccb_h,
3635                                                    xpt_links.stqe);
3636
3637                                 splx(ospl);
3638                                 continue;
3639                         } else {
3640                                 /*
3641                                  * Consume a high power slot while
3642                                  * this ccb runs.
3643                                  */
3644                                 num_highpower--;
3645                         }
3646                 }
3647                 devq->active_dev = device;
3648                 cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
3649
3650                 cam_ccbq_send_ccb(&device->ccbq, work_ccb);
3651                 splx(ospl);
3652
3653                 devq->send_openings--;
3654                 devq->send_active++;
3655
3656                 if (device->ccbq.queue.entries > 0)
3657                         xpt_schedule_dev_sendq(bus, device);
3658
3659                 if (work_ccb && (work_ccb->ccb_h.flags & CAM_DEV_QFREEZE) != 0){
3660                         /*
3661                          * The client wants to freeze the queue
3662                          * after this CCB is sent.
3663                          */
3664                         ospl = splcam();
3665                         device->qfrozen_cnt++;
3666                         splx(ospl);
3667                 }
3668
3669                 splx(s);
3670
3671                 if ((device->inq_flags & SID_CmdQue) != 0)
3672                         work_ccb->ccb_h.flags |= CAM_TAG_ACTION_VALID;
3673                 else
3674                         /*
3675                          * Clear this in case of a retried CCB that failed
3676                          * due to a rejected tag.
3677                          */
3678                         work_ccb->ccb_h.flags &= ~CAM_TAG_ACTION_VALID;
3679
3680                 /*
3681                  * Device queues can be shared among multiple sim instances
3682                  * that reside on different busses.  Use the SIM in the queue
3683                  * CCB's path, rather than the one in the bus that was passed
3684                  * into this function.
3685                  */
3686                 sim = work_ccb->ccb_h.path->bus->sim;
3687                 (*(sim->sim_action))(sim, work_ccb);
3688
3689                 ospl = splcam();
3690                 devq->active_dev = NULL;
3691                 splx(ospl);
3692                 /* Raise IPL for possible insertion and test at top of loop */
3693                 s = splsoftcam();
3694         }
3695         splx(s);
3696         s = splcam();
3697         devq->send_queue.qfrozen_cnt--;
3698         splx(s);
3699 }
3700
3701 /*
3702  * This function merges stuff from the slave ccb into the master ccb, while
3703  * keeping important fields in the master ccb constant.
3704  */
3705 void
3706 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
3707 {
3708         /*
3709          * Pull fields that are valid for peripheral drivers to set
3710          * into the master CCB along with the CCB "payload".
3711          */
3712         master_ccb->ccb_h.retry_count = slave_ccb->ccb_h.retry_count;
3713         master_ccb->ccb_h.func_code = slave_ccb->ccb_h.func_code;
3714         master_ccb->ccb_h.timeout = slave_ccb->ccb_h.timeout;
3715         master_ccb->ccb_h.flags = slave_ccb->ccb_h.flags;
3716         bcopy(&(&slave_ccb->ccb_h)[1], &(&master_ccb->ccb_h)[1],
3717               sizeof(union ccb) - sizeof(struct ccb_hdr));
3718 }
3719
3720 void
3721 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
3722 {
3723         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
3724         ccb_h->pinfo.priority = priority;
3725         ccb_h->path = path;
3726         ccb_h->path_id = path->bus->path_id;
3727         if (path->target)
3728                 ccb_h->target_id = path->target->target_id;
3729         else
3730                 ccb_h->target_id = CAM_TARGET_WILDCARD;
3731         if (path->device) {
3732                 ccb_h->target_lun = path->device->lun_id;
3733                 ccb_h->pinfo.generation = ++path->device->ccbq.queue.generation;
3734         } else {
3735                 ccb_h->target_lun = CAM_TARGET_WILDCARD;
3736         }
3737         ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
3738         ccb_h->flags = 0;
3739 }
3740
3741 /* Path manipulation functions */
3742 cam_status
3743 xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
3744                 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3745 {
3746         struct     cam_path *path;
3747         cam_status status;
3748
3749         path = (struct cam_path *)malloc(sizeof(*path), M_DEVBUF, M_NOWAIT);
3750
3751         if (path == NULL) {
3752                 status = CAM_RESRC_UNAVAIL;
3753                 return(status);
3754         }
3755         status = xpt_compile_path(path, perph, path_id, target_id, lun_id);
3756         if (status != CAM_REQ_CMP) {
3757                 free(path, M_DEVBUF);
3758                 path = NULL;
3759         }
3760         *new_path_ptr = path;
3761         return (status);
3762 }
3763
3764 static cam_status
3765 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
3766                  path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
3767 {
3768         struct       cam_eb *bus;
3769         struct       cam_et *target;
3770         struct       cam_ed *device;
3771         cam_status   status;
3772         int          s;
3773
3774         status = CAM_REQ_CMP;   /* Completed without error */
3775         target = NULL;          /* Wildcarded */
3776         device = NULL;          /* Wildcarded */
3777
3778         /*
3779          * We will potentially modify the EDT, so block interrupts
3780          * that may attempt to create cam paths.
3781          */
3782         s = splcam();
3783         bus = xpt_find_bus(path_id);
3784         if (bus == NULL) {
3785                 status = CAM_PATH_INVALID;
3786         } else {
3787                 target = xpt_find_target(bus, target_id);
3788                 if (target == NULL) {
3789                         /* Create one */
3790                         struct cam_et *new_target;
3791
3792                         new_target = xpt_alloc_target(bus, target_id);
3793                         if (new_target == NULL) {
3794                                 status = CAM_RESRC_UNAVAIL;
3795                         } else {
3796                                 target = new_target;
3797                         }
3798                 }
3799                 if (target != NULL) {
3800                         device = xpt_find_device(target, lun_id);
3801                         if (device == NULL) {
3802                                 /* Create one */
3803                                 struct cam_ed *new_device;
3804
3805                                 new_device = xpt_alloc_device(bus,
3806                                                               target,
3807                                                               lun_id);
3808                                 if (new_device == NULL) {
3809                                         status = CAM_RESRC_UNAVAIL;
3810                                 } else {
3811                                         device = new_device;
3812                                 }
3813                         }
3814                 }
3815         }
3816         splx(s);
3817
3818         /*
3819          * Only touch the user's data if we are successful.
3820          */
3821         if (status == CAM_REQ_CMP) {
3822                 new_path->periph = perph;
3823                 new_path->bus = bus;
3824                 new_path->target = target;
3825                 new_path->device = device;
3826                 CAM_DEBUG(new_path, CAM_DEBUG_TRACE, ("xpt_compile_path\n"));
3827         } else {
3828                 if (device != NULL)
3829                         xpt_release_device(bus, target, device);
3830                 if (target != NULL)
3831                         xpt_release_target(bus, target);
3832                 if (bus != NULL)
3833                         xpt_release_bus(bus);
3834         }
3835         return (status);
3836 }
3837
3838 static void
3839 xpt_release_path(struct cam_path *path)
3840 {
3841         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_release_path\n"));
3842         if (path->device != NULL) {
3843                 xpt_release_device(path->bus, path->target, path->device);
3844                 path->device = NULL;
3845         }
3846         if (path->target != NULL) {
3847                 xpt_release_target(path->bus, path->target);
3848                 path->target = NULL;
3849         }
3850         if (path->bus != NULL) {
3851                 xpt_release_bus(path->bus);
3852                 path->bus = NULL;
3853         }
3854 }
3855
3856 void
3857 xpt_free_path(struct cam_path *path)
3858 {
3859         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
3860         xpt_release_path(path);
3861         free(path, M_DEVBUF);
3862 }
3863
3864
3865 /*
3866  * Return -1 for failure, 0 for exact match, 1 for match with wildcards
3867  * in path1, 2 for match with wildcards in path2.
3868  */
3869 int
3870 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
3871 {
3872         int retval = 0;
3873
3874         if (path1->bus != path2->bus) {
3875                 if (path1->bus->path_id == CAM_BUS_WILDCARD)
3876                         retval = 1;
3877                 else if (path2->bus->path_id == CAM_BUS_WILDCARD)
3878                         retval = 2;
3879                 else
3880                         return (-1);
3881         }
3882         if (path1->target != path2->target) {
3883                 if (path1->target->target_id == CAM_TARGET_WILDCARD) {
3884                         if (retval == 0)
3885                                 retval = 1;
3886                 } else if (path2->target->target_id == CAM_TARGET_WILDCARD)
3887                         retval = 2;
3888                 else
3889                         return (-1);
3890         }
3891         if (path1->device != path2->device) {
3892                 if (path1->device->lun_id == CAM_LUN_WILDCARD) {
3893                         if (retval == 0)
3894                                 retval = 1;
3895                 } else if (path2->device->lun_id == CAM_LUN_WILDCARD)
3896                         retval = 2;
3897                 else
3898                         return (-1);
3899         }
3900         return (retval);
3901 }
3902
3903 void
3904 xpt_print_path(struct cam_path *path)
3905 {
3906         if (path == NULL)
3907                 printf("(nopath): ");
3908         else {
3909                 if (path->periph != NULL)
3910                         printf("(%s%d:", path->periph->periph_name,
3911                                path->periph->unit_number);
3912                 else
3913                         printf("(noperiph:");
3914
3915                 if (path->bus != NULL)
3916                         printf("%s%d:%d:", path->bus->sim->sim_name,
3917                                path->bus->sim->unit_number,
3918                                path->bus->sim->bus_id);
3919                 else
3920                         printf("nobus:");
3921
3922                 if (path->target != NULL)
3923                         printf("%d:", path->target->target_id);
3924                 else
3925                         printf("X:");
3926
3927                 if (path->device != NULL)
3928                         printf("%d): ", path->device->lun_id);
3929                 else
3930                         printf("X): ");
3931         }
3932 }
3933
3934 path_id_t
3935 xpt_path_path_id(struct cam_path *path)
3936 {
3937         return(path->bus->path_id);
3938 }
3939
3940 target_id_t
3941 xpt_path_target_id(struct cam_path *path)
3942 {
3943         if (path->target != NULL)
3944                 return (path->target->target_id);
3945         else
3946                 return (CAM_TARGET_WILDCARD);
3947 }
3948
3949 lun_id_t
3950 xpt_path_lun_id(struct cam_path *path)
3951 {
3952         if (path->device != NULL)
3953                 return (path->device->lun_id);
3954         else
3955                 return (CAM_LUN_WILDCARD);
3956 }
3957
3958 struct cam_sim *
3959 xpt_path_sim(struct cam_path *path)
3960 {
3961         return (path->bus->sim);
3962 }
3963
3964 struct cam_periph*
3965 xpt_path_periph(struct cam_path *path)
3966 {
3967         return (path->periph);
3968 }
3969
3970 /*
3971  * Release a CAM control block for the caller.  Remit the cost of the structure
3972  * to the device referenced by the path.  If the this device had no 'credits'
3973  * and peripheral drivers have registered async callbacks for this notification
3974  * call them now.
3975  */
3976 void
3977 xpt_release_ccb(union ccb *free_ccb)
3978 {
3979         int      s;
3980         struct   cam_path *path;
3981         struct   cam_ed *device;
3982         struct   cam_eb *bus;
3983
3984         CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
3985         path = free_ccb->ccb_h.path;
3986         device = path->device;
3987         bus = path->bus;
3988         s = splsoftcam();
3989         cam_ccbq_release_opening(&device->ccbq);
3990         if (xpt_ccb_count > xpt_max_ccbs) {
3991                 xpt_free_ccb(free_ccb);
3992                 xpt_ccb_count--;
3993         } else {
3994                 SLIST_INSERT_HEAD(&ccb_freeq, &free_ccb->ccb_h, xpt_links.sle);
3995         }
3996         bus->sim->devq->alloc_openings++;
3997         bus->sim->devq->alloc_active--;
3998         /* XXX Turn this into an inline function - xpt_run_device?? */
3999         if ((device_is_alloc_queued(device) == 0)
4000          && (device->drvq.entries > 0)) {
4001                 xpt_schedule_dev_allocq(bus, device);
4002         }
4003         splx(s);
4004         if (dev_allocq_is_runnable(bus->sim->devq))
4005                 xpt_run_dev_allocq(bus);
4006 }
4007
4008 /* Functions accessed by SIM drivers */
4009
4010 /*
4011  * A sim structure, listing the SIM entry points and instance
4012  * identification info is passed to xpt_bus_register to hook the SIM
4013  * into the CAM framework.  xpt_bus_register creates a cam_eb entry
4014  * for this new bus and places it in the array of busses and assigns
4015  * it a path_id.  The path_id may be influenced by "hard wiring"
4016  * information specified by the user.  Once interrupt services are
4017  * availible, the bus will be probed.
4018  */
4019 int32_t
4020 xpt_bus_register(struct cam_sim *sim, u_int32_t bus)
4021 {
4022         struct cam_eb *new_bus;
4023         struct cam_eb *old_bus;
4024         struct ccb_pathinq cpi;
4025         int s;
4026
4027         sim->bus_id = bus;
4028         new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
4029                                           M_DEVBUF, M_NOWAIT);
4030         if (new_bus == NULL) {
4031                 /* Couldn't satisfy request */
4032                 return (CAM_RESRC_UNAVAIL);
4033         }
4034
4035         if (strcmp(sim->sim_name, "xpt") != 0) {
4036
4037                 sim->path_id =
4038                     xptpathid(sim->sim_name, sim->unit_number, sim->bus_id);
4039         }
4040
4041         TAILQ_INIT(&new_bus->et_entries);
4042         new_bus->path_id = sim->path_id;
4043         new_bus->sim = sim;
4044         timevalclear(&new_bus->last_reset);
4045         new_bus->flags = 0;
4046         new_bus->refcount = 1;  /* Held until a bus_deregister event */
4047         new_bus->generation = 0;
4048         s = splcam();
4049         old_bus = TAILQ_FIRST(&xpt_busses);
4050         while (old_bus != NULL
4051             && old_bus->path_id < new_bus->path_id)
4052                 old_bus = TAILQ_NEXT(old_bus, links);
4053         if (old_bus != NULL)
4054                 TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
4055         else
4056                 TAILQ_INSERT_TAIL(&xpt_busses, new_bus, links);
4057         bus_generation++;
4058         splx(s);
4059
4060         /* Notify interested parties */
4061         if (sim->path_id != CAM_XPT_PATH_ID) {
4062                 struct cam_path path;
4063
4064                 xpt_compile_path(&path, /*periph*/NULL, sim->path_id,
4065                                  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4066                 xpt_setup_ccb(&cpi.ccb_h, &path, /*priority*/1);
4067                 cpi.ccb_h.func_code = XPT_PATH_INQ;
4068                 xpt_action((union ccb *)&cpi);
4069                 xpt_async(AC_PATH_REGISTERED, xpt_periph->path, &cpi);
4070                 xpt_release_path(&path);
4071         }
4072         return (CAM_SUCCESS);
4073 }
4074
4075 int32_t
4076 xpt_bus_deregister(path_id_t pathid)
4077 {
4078         struct cam_path bus_path;
4079         cam_status status;
4080
4081         status = xpt_compile_path(&bus_path, NULL, pathid,
4082                                   CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4083         if (status != CAM_REQ_CMP)
4084                 return (status);
4085
4086         xpt_async(AC_LOST_DEVICE, &bus_path, NULL);
4087         xpt_async(AC_PATH_DEREGISTERED, &bus_path, NULL);
4088
4089         /* Release the reference count held while registered. */
4090         xpt_release_bus(bus_path.bus);
4091         xpt_release_path(&bus_path);
4092
4093         return (CAM_REQ_CMP);
4094 }
4095
4096 static path_id_t
4097 xptnextfreepathid(void)
4098 {
4099         struct cam_eb *bus;
4100         path_id_t pathid;
4101         char *strval;
4102
4103         pathid = 0;
4104         bus = TAILQ_FIRST(&xpt_busses);
4105 retry:
4106         /* Find an unoccupied pathid */
4107         while (bus != NULL
4108             && bus->path_id <= pathid) {
4109                 if (bus->path_id == pathid)
4110                         pathid++;
4111                 bus = TAILQ_NEXT(bus, links);
4112         }
4113
4114         /*
4115          * Ensure that this pathid is not reserved for
4116          * a bus that may be registered in the future.
4117          */
4118         if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
4119                 ++pathid;
4120                 /* Start the search over */
4121                 goto retry;
4122         }
4123         return (pathid);
4124 }
4125
4126 static path_id_t
4127 xptpathid(const char *sim_name, int sim_unit, int sim_bus)
4128 {
4129         path_id_t pathid;
4130         int i, dunit, val;
4131         char buf[32], *strval;
4132
4133         pathid = CAM_XPT_PATH_ID;
4134         snprintf(buf, sizeof(buf), "%s%d", sim_name, sim_unit);
4135         i = -1;
4136         while ((i = resource_locate(i, "scbus")) != -1) {
4137                 dunit = resource_query_unit(i);
4138                 if (dunit < 0)          /* unwired?! */
4139                         continue;
4140                 if (resource_string_value("scbus", dunit, "at", &strval) != 0)
4141                         continue;
4142                 if (strcmp(buf, strval) != 0)
4143                         continue;
4144                 if (resource_int_value("scbus", dunit, "bus", &val) == 0) {
4145                         if (sim_bus == val) {
4146                                 pathid = dunit;
4147                                 break;
4148                         }
4149                 } else if (sim_bus == 0) {
4150                         /* Unspecified matches bus 0 */
4151                         pathid = dunit;
4152                         break;
4153                 } else {
4154                         printf("Ambiguous scbus configuration for %s%d "
4155                                "bus %d, cannot wire down.  The kernel "
4156                                "config entry for scbus%d should "
4157                                "specify a controller bus.\n"
4158                                "Scbus will be assigned dynamically.\n",
4159                                sim_name, sim_unit, sim_bus, dunit);
4160                         break;
4161                 }
4162         }
4163
4164         if (pathid == CAM_XPT_PATH_ID)
4165                 pathid = xptnextfreepathid();
4166         return (pathid);
4167 }
4168
4169 void
4170 xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
4171 {
4172         struct cam_eb *bus;
4173         struct cam_et *target, *next_target;
4174         struct cam_ed *device, *next_device;
4175         int s;
4176
4177         CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_async\n"));
4178
4179         /*
4180          * Most async events come from a CAM interrupt context.  In
4181          * a few cases, the error recovery code at the peripheral layer,
4182          * which may run from our SWI or a process context, may signal
4183          * deferred events with a call to xpt_async. Ensure async
4184          * notifications are serialized by blocking cam interrupts.
4185          */
4186         s = splcam();
4187
4188         bus = path->bus;
4189
4190         if (async_code == AC_BUS_RESET) {
4191                 int s;
4192
4193                 s = splclock();
4194                 /* Update our notion of when the last reset occurred */
4195                 microtime(&bus->last_reset);
4196                 splx(s);
4197         }
4198
4199         for (target = TAILQ_FIRST(&bus->et_entries);
4200              target != NULL;
4201              target = next_target) {
4202
4203                 next_target = TAILQ_NEXT(target, links);
4204
4205                 if (path->target != target
4206                  && path->target->target_id != CAM_TARGET_WILDCARD)
4207                         continue;
4208
4209                 if (async_code == AC_SENT_BDR) {
4210                         int s;
4211
4212                         /* Update our notion of when the last reset occurred */
4213                         s = splclock();
4214                         microtime(&path->target->last_reset);
4215                         splx(s);
4216                 }
4217
4218                 for (device = TAILQ_FIRST(&target->ed_entries);
4219                      device != NULL;
4220                      device = next_device) {
4221                         cam_status status;
4222                         struct cam_path newpath;
4223
4224                         next_device = TAILQ_NEXT(device, links);
4225
4226                         if (path->device != device
4227                          && path->device->lun_id != CAM_LUN_WILDCARD)
4228                                 continue;
4229
4230                         /*
4231                          * We need our own path with wildcards expanded to
4232                          * handle certain types of events.
4233                          */
4234                         if ((async_code == AC_SENT_BDR)
4235                          || (async_code == AC_BUS_RESET)
4236                          || (async_code == AC_INQ_CHANGED))
4237                                 status = xpt_compile_path(&newpath, NULL,
4238                                                           bus->path_id,
4239                                                           target->target_id,
4240                                                           device->lun_id);
4241                         else
4242                                 status = CAM_REQ_CMP_ERR;
4243
4244                         if (status == CAM_REQ_CMP) {
4245
4246                                 /*
4247                                  * Allow transfer negotiation to occur in a
4248                                  * tag free environment.
4249                                  */
4250                                 if (async_code == AC_SENT_BDR
4251                                   || async_code == AC_BUS_RESET)
4252                                         xpt_toggle_tags(&newpath);
4253
4254                                 if (async_code == AC_INQ_CHANGED) {
4255                                         /*
4256                                          * We've sent a start unit command, or
4257                                          * something similar to a device that
4258                                          * may have caused its inquiry data to
4259                                          * change. So we re-scan the device to
4260                                          * refresh the inquiry data for it.
4261                                          */
4262                                         xpt_scan_lun(newpath.periph, &newpath,
4263                                                      CAM_EXPECT_INQ_CHANGE,
4264                                                      NULL);
4265                                 }
4266                                 xpt_release_path(&newpath);
4267                         } else if (async_code == AC_LOST_DEVICE) {
4268                                 device->flags |= CAM_DEV_UNCONFIGURED;
4269                         } else if (async_code == AC_TRANSFER_NEG) {
4270                                 struct ccb_trans_settings *settings;
4271
4272                                 settings =
4273                                     (struct ccb_trans_settings *)async_arg;
4274                                 xpt_set_transfer_settings(settings, device,
4275                                                           /*async_update*/TRUE);
4276                         }
4277
4278                         xpt_async_bcast(&device->asyncs,
4279                                         async_code,
4280                                         path,
4281                                         async_arg);
4282                 }
4283         }
4284
4285         /*
4286          * If this wasn't a fully wildcarded async, tell all
4287          * clients that want all async events.
4288          */
4289         if (bus != xpt_periph->path->bus)
4290                 xpt_async_bcast(&xpt_periph->path->device->asyncs, async_code,
4291                                 path, async_arg);
4292         splx(s);
4293 }
4294
4295 static void
4296 xpt_async_bcast(struct async_list *async_head,
4297                 u_int32_t async_code,
4298                 struct cam_path *path, void *async_arg)
4299 {
4300         struct async_node *cur_entry;
4301
4302         cur_entry = SLIST_FIRST(async_head);
4303         while (cur_entry != NULL) {
4304                 struct async_node *next_entry;
4305                 /*
4306                  * Grab the next list entry before we call the current
4307                  * entry's callback.  This is because the callback function
4308                  * can delete its async callback entry.
4309                  */
4310                 next_entry = SLIST_NEXT(cur_entry, links);
4311                 if ((cur_entry->event_enable & async_code) != 0)
4312                         cur_entry->callback(cur_entry->callback_arg,
4313                                             async_code, path,
4314                                             async_arg);
4315                 cur_entry = next_entry;
4316         }
4317 }
4318
4319 u_int32_t
4320 xpt_freeze_devq(struct cam_path *path, u_int count)
4321 {
4322         int s;
4323         struct ccb_hdr *ccbh;
4324
4325         s = splcam();
4326         path->device->qfrozen_cnt += count;
4327
4328         /*
4329          * Mark the last CCB in the queue as needing
4330          * to be requeued if the driver hasn't
4331          * changed it's state yet.  This fixes a race
4332          * where a ccb is just about to be queued to
4333          * a controller driver when it's interrupt routine
4334          * freezes the queue.  To completly close the
4335          * hole, controller drives must check to see
4336          * if a ccb's status is still CAM_REQ_INPROG
4337          * under spl protection just before they queue
4338          * the CCB.  See ahc_action/ahc_freeze_devq for
4339          * an example.
4340          */
4341         ccbh = TAILQ_LAST(&path->device->ccbq.active_ccbs, ccb_hdr_tailq);
4342         if (ccbh && ccbh->status == CAM_REQ_INPROG)
4343                 ccbh->status = CAM_REQUEUE_REQ;
4344         splx(s);
4345         return (path->device->qfrozen_cnt);
4346 }
4347
4348 u_int32_t
4349 xpt_freeze_simq(struct cam_sim *sim, u_int count)
4350 {
4351         sim->devq->send_queue.qfrozen_cnt += count;
4352         if (sim->devq->active_dev != NULL) {
4353                 struct ccb_hdr *ccbh;
4354
4355                 ccbh = TAILQ_LAST(&sim->devq->active_dev->ccbq.active_ccbs,
4356                                   ccb_hdr_tailq);
4357                 if (ccbh && ccbh->status == CAM_REQ_INPROG)
4358                         ccbh->status = CAM_REQUEUE_REQ;
4359         }
4360         return (sim->devq->send_queue.qfrozen_cnt);
4361 }
4362
4363 static void
4364 xpt_release_devq_timeout(void *arg)
4365 {
4366         struct cam_ed *device;
4367
4368         device = (struct cam_ed *)arg;
4369
4370         xpt_release_devq_device(device, /*count*/1, /*run_queue*/TRUE);
4371 }
4372
4373 void
4374 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
4375 {
4376         xpt_release_devq_device(path->device, count, run_queue);
4377 }
4378
4379 static void
4380 xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
4381 {
4382         int     rundevq;
4383         int     s0, s1;
4384
4385         rundevq = 0;
4386         s0 = splsoftcam();
4387         s1 = splcam();
4388         if (dev->qfrozen_cnt > 0) {
4389
4390                 count = (count > dev->qfrozen_cnt) ? dev->qfrozen_cnt : count;
4391                 dev->qfrozen_cnt -= count;
4392                 if (dev->qfrozen_cnt == 0) {
4393
4394                         /*
4395                          * No longer need to wait for a successful
4396                          * command completion.
4397                          */
4398                         dev->flags &= ~CAM_DEV_REL_ON_COMPLETE;
4399
4400                         /*
4401                          * Remove any timeouts that might be scheduled
4402                          * to release this queue.
4403                          */
4404                         if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
4405                                 untimeout(xpt_release_devq_timeout, dev,
4406                                           dev->c_handle);
4407                                 dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
4408                         }
4409
4410                         /*
4411                          * Now that we are unfrozen schedule the
4412                          * device so any pending transactions are
4413                          * run.
4414                          */
4415                         if ((dev->ccbq.queue.entries > 0)
4416                          && (xpt_schedule_dev_sendq(dev->target->bus, dev))
4417                          && (run_queue != 0)) {
4418                                 rundevq = 1;
4419                         }
4420                 }
4421         }
4422         splx(s1);
4423         if (rundevq != 0)
4424                 xpt_run_dev_sendq(dev->target->bus);
4425         splx(s0);
4426 }
4427
4428 void
4429 xpt_release_simq(struct cam_sim *sim, int run_queue)
4430 {
4431         int     s;
4432         struct  camq *sendq;
4433
4434         sendq = &(sim->devq->send_queue);
4435         s = splcam();
4436         if (sendq->qfrozen_cnt > 0) {
4437
4438                 sendq->qfrozen_cnt--;
4439                 if (sendq->qfrozen_cnt == 0) {
4440                         struct cam_eb *bus;
4441
4442                         /*
4443                          * If there is a timeout scheduled to release this
4444                          * sim queue, remove it.  The queue frozen count is
4445                          * already at 0.
4446                          */
4447                         if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
4448                                 untimeout(xpt_release_simq_timeout, sim,
4449                                           sim->c_handle);
4450                                 sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
4451                         }
4452                         bus = xpt_find_bus(sim->path_id);
4453                         splx(s);
4454
4455                         if (run_queue) {
4456                                 /*
4457                                  * Now that we are unfrozen run the send queue.
4458                                  */
4459                                 xpt_run_dev_sendq(bus);
4460                         }
4461                         xpt_release_bus(bus);
4462                 } else
4463                         splx(s);
4464         } else
4465                 splx(s);
4466 }
4467
4468 static void
4469 xpt_release_simq_timeout(void *arg)
4470 {
4471         struct cam_sim *sim;
4472
4473         sim = (struct cam_sim *)arg;
4474         xpt_release_simq(sim, /* run_queue */ TRUE);
4475 }
4476
4477 void
4478 xpt_done(union ccb *done_ccb)
4479 {
4480         int s;
4481
4482         s = splcam();
4483
4484         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_done\n"));
4485         if ((done_ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0) {
4486                 /*
4487                  * Queue up the request for handling by our SWI handler
4488                  * any of the "non-immediate" type of ccbs.
4489                  */
4490                 switch (done_ccb->ccb_h.path->periph->type) {
4491                 case CAM_PERIPH_BIO:
4492                         TAILQ_INSERT_TAIL(&cam_bioq, &done_ccb->ccb_h,
4493                                           sim_links.tqe);
4494                         done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4495                         setsoftcambio();
4496                         break;
4497                 case CAM_PERIPH_NET:
4498                         TAILQ_INSERT_TAIL(&cam_netq, &done_ccb->ccb_h,
4499                                           sim_links.tqe);
4500                         done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
4501                         setsoftcamnet();
4502                         break;
4503                 }
4504         }
4505         splx(s);
4506 }
4507
4508 union ccb *
4509 xpt_alloc_ccb()
4510 {
4511         union ccb *new_ccb;
4512
4513         new_ccb = malloc(sizeof(*new_ccb), M_DEVBUF, M_WAITOK);
4514         return (new_ccb);
4515 }
4516
4517 void
4518 xpt_free_ccb(union ccb *free_ccb)
4519 {
4520         free(free_ccb, M_DEVBUF);
4521 }
4522
4523
4524
4525 /* Private XPT functions */
4526
4527 /*
4528  * Get a CAM control block for the caller. Charge the structure to the device
4529  * referenced by the path.  If the this device has no 'credits' then the
4530  * device already has the maximum number of outstanding operations under way
4531  * and we return NULL. If we don't have sufficient resources to allocate more
4532  * ccbs, we also return NULL.
4533  */
4534 static union ccb *
4535 xpt_get_ccb(struct cam_ed *device)
4536 {
4537         union ccb *new_ccb;
4538         int s;
4539
4540         s = splsoftcam();
4541         if ((new_ccb = (union ccb *)ccb_freeq.slh_first) == NULL) {
4542                 new_ccb = malloc(sizeof(*new_ccb), M_DEVBUF, M_NOWAIT);
4543                 if (new_ccb == NULL) {
4544                         splx(s);
4545                         return (NULL);
4546                 }
4547                 callout_handle_init(&new_ccb->ccb_h.timeout_ch);
4548                 SLIST_INSERT_HEAD(&ccb_freeq, &new_ccb->ccb_h,
4549                                   xpt_links.sle);
4550                 xpt_ccb_count++;
4551         }
4552         cam_ccbq_take_opening(&device->ccbq);
4553         SLIST_REMOVE_HEAD(&ccb_freeq, xpt_links.sle);
4554         splx(s);
4555         return (new_ccb);
4556 }
4557
4558 static void
4559 xpt_release_bus(struct cam_eb *bus)
4560 {
4561         int s;
4562
4563         s = splcam();
4564         if ((--bus->refcount == 0)
4565          && (TAILQ_FIRST(&bus->et_entries) == NULL)) {
4566                 TAILQ_REMOVE(&xpt_busses, bus, links);
4567                 bus_generation++;
4568                 splx(s);
4569                 free(bus, M_DEVBUF);
4570         } else
4571                 splx(s);
4572 }
4573
4574 static struct cam_et *
4575 xpt_alloc_target(struct cam_eb *bus, target_id_t target_id)
4576 {
4577         struct cam_et *target;
4578
4579         target = (struct cam_et *)malloc(sizeof(*target), M_DEVBUF, M_NOWAIT);
4580         if (target != NULL) {
4581                 struct cam_et *cur_target;
4582
4583                 TAILQ_INIT(&target->ed_entries);
4584                 target->bus = bus;
4585                 target->target_id = target_id;
4586                 target->refcount = 1;
4587                 target->generation = 0;
4588                 timevalclear(&target->last_reset);
4589                 /*
4590                  * Hold a reference to our parent bus so it
4591                  * will not go away before we do.
4592                  */
4593                 bus->refcount++;
4594
4595                 /* Insertion sort into our bus's target list */
4596                 cur_target = TAILQ_FIRST(&bus->et_entries);
4597                 while (cur_target != NULL && cur_target->target_id < target_id)
4598                         cur_target = TAILQ_NEXT(cur_target, links);
4599
4600                 if (cur_target != NULL) {
4601                         TAILQ_INSERT_BEFORE(cur_target, target, links);
4602                 } else {
4603                         TAILQ_INSERT_TAIL(&bus->et_entries, target, links);
4604                 }
4605                 bus->generation++;
4606         }
4607         return (target);
4608 }
4609
4610 static void
4611 xpt_release_target(struct cam_eb *bus, struct cam_et *target)
4612 {
4613         int s;
4614
4615         s = splcam();
4616         if ((--target->refcount == 0)
4617          && (TAILQ_FIRST(&target->ed_entries) == NULL)) {
4618                 TAILQ_REMOVE(&bus->et_entries, target, links);
4619                 bus->generation++;
4620                 splx(s);
4621                 free(target, M_DEVBUF);
4622                 xpt_release_bus(bus);
4623         } else
4624                 splx(s);
4625 }
4626
4627 static struct cam_ed *
4628 xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
4629 {
4630         struct     cam_ed *device;
4631         struct     cam_devq *devq;
4632         cam_status status;
4633
4634         /* Make space for us in the device queue on our bus */
4635         devq = bus->sim->devq;
4636         status = cam_devq_resize(devq, devq->alloc_queue.array_size + 1);
4637
4638         if (status != CAM_REQ_CMP) {
4639                 device = NULL;
4640         } else {
4641                 device = (struct cam_ed *)malloc(sizeof(*device),
4642                                                  M_DEVBUF, M_NOWAIT);
4643         }
4644
4645         if (device != NULL) {
4646                 struct cam_ed *cur_device;
4647
4648                 cam_init_pinfo(&device->alloc_ccb_entry.pinfo);
4649                 device->alloc_ccb_entry.device = device;
4650                 cam_init_pinfo(&device->send_ccb_entry.pinfo);
4651                 device->send_ccb_entry.device = device;
4652                 device->target = target;
4653                 device->lun_id = lun_id;
4654                 /* Initialize our queues */
4655                 if (camq_init(&device->drvq, 0) != 0) {
4656                         free(device, M_DEVBUF);
4657                         return (NULL);
4658                 }
4659                 if (cam_ccbq_init(&device->ccbq,
4660                                   bus->sim->max_dev_openings) != 0) {
4661                         camq_fini(&device->drvq);
4662                         free(device, M_DEVBUF);
4663                         return (NULL);
4664                 }
4665                 SLIST_INIT(&device->asyncs);
4666                 SLIST_INIT(&device->periphs);
4667                 device->generation = 0;
4668                 device->owner = NULL;
4669                 /*
4670                  * Take the default quirk entry until we have inquiry
4671                  * data and can determine a better quirk to use.
4672                  */
4673                 device->quirk = &xpt_quirk_table[xpt_quirk_table_size - 1];
4674                 bzero(&device->inq_data, sizeof(device->inq_data));
4675                 device->inq_flags = 0;
4676                 device->queue_flags = 0;
4677                 device->serial_num = NULL;
4678                 device->serial_num_len = 0;
4679                 device->qfrozen_cnt = 0;
4680                 device->flags = CAM_DEV_UNCONFIGURED;
4681                 device->tag_delay_count = 0;
4682                 device->refcount = 1;
4683                 callout_handle_init(&device->c_handle);
4684
4685                 /*
4686                  * Hold a reference to our parent target so it
4687                  * will not go away before we do.
4688                  */
4689                 target->refcount++;
4690
4691                 /*
4692                  * XXX should be limited by number of CCBs this bus can
4693                  * do.
4694                  */
4695                 xpt_max_ccbs += device->ccbq.devq_openings;
4696                 /* Insertion sort into our target's device list */
4697                 cur_device = TAILQ_FIRST(&target->ed_entries);
4698                 while (cur_device != NULL && cur_device->lun_id < lun_id)
4699                         cur_device = TAILQ_NEXT(cur_device, links);
4700                 if (cur_device != NULL) {
4701                         TAILQ_INSERT_BEFORE(cur_device, device, links);
4702                 } else {
4703                         TAILQ_INSERT_TAIL(&target->ed_entries, device, links);
4704                 }
4705                 target->generation++;
4706         }
4707         return (device);
4708 }
4709
4710 static void
4711 xpt_release_device(struct cam_eb *bus, struct cam_et *target,
4712                    struct cam_ed *device)
4713 {
4714         int s;
4715
4716         s = splcam();
4717         if ((--device->refcount == 0)
4718          && ((device->flags & CAM_DEV_UNCONFIGURED) != 0)) {
4719                 struct cam_devq *devq;
4720
4721                 if (device->alloc_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX
4722                  || device->send_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX)
4723                         panic("Removing device while still queued for ccbs");
4724
4725                 if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
4726                                 untimeout(xpt_release_devq_timeout, device,
4727                                           device->c_handle);
4728
4729                 TAILQ_REMOVE(&target->ed_entries, device,links);
4730                 target->generation++;
4731                 xpt_max_ccbs -= device->ccbq.devq_openings;
4732                 /* Release our slot in the devq */
4733                 devq = bus->sim->devq;
4734                 cam_devq_resize(devq, devq->alloc_queue.array_size - 1);
4735                 splx(s);
4736                 free(device, M_DEVBUF);
4737                 xpt_release_target(bus, target);
4738         } else
4739                 splx(s);
4740 }
4741
4742 static u_int32_t
4743 xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
4744 {
4745         int     s;
4746         int     diff;
4747         int     result;
4748         struct  cam_ed *dev;
4749
4750         dev = path->device;
4751         s = splsoftcam();
4752
4753         diff = newopenings - (dev->ccbq.dev_active + dev->ccbq.dev_openings);
4754         result = cam_ccbq_resize(&dev->ccbq, newopenings);
4755         if (result == CAM_REQ_CMP && (diff < 0)) {
4756                 dev->flags |= CAM_DEV_RESIZE_QUEUE_NEEDED;
4757         }
4758         /* Adjust the global limit */
4759         xpt_max_ccbs += diff;
4760         splx(s);
4761         return (result);
4762 }
4763
4764 static struct cam_eb *
4765 xpt_find_bus(path_id_t path_id)
4766 {
4767         struct cam_eb *bus;
4768
4769         for (bus = TAILQ_FIRST(&xpt_busses);
4770              bus != NULL;
4771              bus = TAILQ_NEXT(bus, links)) {
4772                 if (bus->path_id == path_id) {
4773                         bus->refcount++;
4774                         break;
4775                 }
4776         }
4777         return (bus);
4778 }
4779
4780 static struct cam_et *
4781 xpt_find_target(struct cam_eb *bus, target_id_t target_id)
4782 {
4783         struct cam_et *target;
4784
4785         for (target = TAILQ_FIRST(&bus->et_entries);
4786              target != NULL;
4787              target = TAILQ_NEXT(target, links)) {
4788                 if (target->target_id == target_id) {
4789                         target->refcount++;
4790                         break;
4791                 }
4792         }
4793         return (target);
4794 }
4795
4796 static struct cam_ed *
4797 xpt_find_device(struct cam_et *target, lun_id_t lun_id)
4798 {
4799         struct cam_ed *device;
4800
4801         for (device = TAILQ_FIRST(&target->ed_entries);
4802              device != NULL;
4803              device = TAILQ_NEXT(device, links)) {
4804                 if (device->lun_id == lun_id) {
4805                         device->refcount++;
4806                         break;
4807                 }
4808         }
4809         return (device);
4810 }
4811
4812 typedef struct {
4813         union   ccb *request_ccb;
4814         struct  ccb_pathinq *cpi;
4815         int     pending_count;
4816 } xpt_scan_bus_info;
4817
4818 /*
4819  * To start a scan, request_ccb is an XPT_SCAN_BUS ccb.
4820  * As the scan progresses, xpt_scan_bus is used as the
4821  * callback on completion function.
4822  */
4823 static void
4824 xpt_scan_bus(struct cam_periph *periph, union ccb *request_ccb)
4825 {
4826         CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE,
4827                   ("xpt_scan_bus\n"));
4828         switch (request_ccb->ccb_h.func_code) {
4829         case XPT_SCAN_BUS:
4830         {
4831                 xpt_scan_bus_info *scan_info;
4832                 union   ccb *work_ccb;
4833                 struct  cam_path *path;
4834                 u_int   i;
4835                 u_int   max_target;
4836                 u_int   initiator_id;
4837
4838                 /* Find out the characteristics of the bus */
4839                 work_ccb = xpt_alloc_ccb();
4840                 xpt_setup_ccb(&work_ccb->ccb_h, request_ccb->ccb_h.path,
4841                               request_ccb->ccb_h.pinfo.priority);
4842                 work_ccb->ccb_h.func_code = XPT_PATH_INQ;
4843                 xpt_action(work_ccb);
4844                 if (work_ccb->ccb_h.status != CAM_REQ_CMP) {
4845                         request_ccb->ccb_h.status = work_ccb->ccb_h.status;
4846                         xpt_free_ccb(work_ccb);
4847                         xpt_done(request_ccb);
4848                         return;
4849                 }
4850
4851                 if ((work_ccb->cpi.hba_misc & PIM_NOINITIATOR) != 0) {
4852                         /*
4853                          * Can't scan the bus on an adapter that
4854                          * cannot perform the initiator role.
4855                          */
4856                         request_ccb->ccb_h.status = CAM_REQ_CMP;
4857                         xpt_free_ccb(work_ccb);
4858                         xpt_done(request_ccb);
4859                         return;
4860                 }
4861
4862                 /* Save some state for use while we probe for devices */
4863                 scan_info = (xpt_scan_bus_info *)
4864                     malloc(sizeof(xpt_scan_bus_info), M_TEMP, M_WAITOK);
4865                 scan_info->request_ccb = request_ccb;
4866                 scan_info->cpi = &work_ccb->cpi;
4867
4868                 /* Cache on our stack so we can work asynchronously */
4869                 max_target = scan_info->cpi->max_target;
4870                 initiator_id = scan_info->cpi->initiator_id;
4871
4872                 /*
4873                  * Don't count the initiator if the
4874                  * initiator is addressable.
4875                  */
4876                 scan_info->pending_count = max_target + 1;
4877                 if (initiator_id <= max_target)
4878                         scan_info->pending_count--;
4879
4880                 for (i = 0; i <= max_target; i++) {
4881                         cam_status status;
4882                         if (i == initiator_id)
4883                                 continue;
4884
4885                         status = xpt_create_path(&path, xpt_periph,
4886                                                  request_ccb->ccb_h.path_id,
4887                                                  i, 0);
4888                         if (status != CAM_REQ_CMP) {
4889                                 printf("xpt_scan_bus: xpt_create_path failed"
4890                                        " with status %#x, bus scan halted\n",
4891                                        status);
4892                                 break;
4893                         }
4894                         work_ccb = xpt_alloc_ccb();
4895                         xpt_setup_ccb(&work_ccb->ccb_h, path,
4896                                       request_ccb->ccb_h.pinfo.priority);
4897                         work_ccb->ccb_h.func_code = XPT_SCAN_LUN;
4898                         work_ccb->ccb_h.cbfcnp = xpt_scan_bus;
4899                         work_ccb->ccb_h.ppriv_ptr0 = scan_info;
4900                         work_ccb->crcn.flags = request_ccb->crcn.flags;
4901 #if 0
4902                         printf("xpt_scan_bus: probing %d:%d:%d\n",
4903                                 request_ccb->ccb_h.path_id, i, 0);
4904 #endif
4905                         xpt_action(work_ccb);
4906                 }
4907                 break;
4908         }
4909         case XPT_SCAN_LUN:
4910         {
4911                 xpt_scan_bus_info *scan_info;
4912                 path_id_t path_id;
4913                 target_id_t target_id;
4914                 lun_id_t lun_id;
4915
4916                 /* Reuse the same CCB to query if a device was really found */
4917                 scan_info = (xpt_scan_bus_info *)request_ccb->ccb_h.ppriv_ptr0;
4918                 xpt_setup_ccb(&request_ccb->ccb_h, request_ccb->ccb_h.path,
4919                               request_ccb->ccb_h.pinfo.priority);
4920                 request_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
4921
4922                 path_id = request_ccb->ccb_h.path_id;
4923                 target_id = request_ccb->ccb_h.target_id;
4924                 lun_id = request_ccb->ccb_h.target_lun;
4925                 xpt_action(request_ccb);
4926
4927 #if 0
4928                 printf("xpt_scan_bus: got back probe from %d:%d:%d\n",
4929                         path_id, target_id, lun_id);
4930 #endif
4931
4932                 if (request_ccb->ccb_h.status != CAM_REQ_CMP) {
4933                         struct cam_ed *device;
4934                         struct cam_et *target;
4935                         int s, phl;
4936
4937                         /*
4938                          * If we already probed lun 0 successfully, or
4939                          * we have additional configured luns on this
4940                          * target that might have "gone away", go onto
4941                          * the next lun.
4942                          */
4943                         target = request_ccb->ccb_h.path->target;
4944                         /*
4945                          * We may touch devices that we don't
4946                          * hold references too, so ensure they
4947                          * don't disappear out from under us.
4948                          * The target above is referenced by the
4949                          * path in the request ccb.
4950                          */
4951                         phl = 0;
4952                         s = splcam();
4953                         device = TAILQ_FIRST(&target->ed_entries);
4954                         if (device != NULL) {
4955                                 phl = device->quirk->quirks & CAM_QUIRK_HILUNS;
4956                                 if (device->lun_id == 0)
4957                                         device = TAILQ_NEXT(device, links);
4958                         }
4959                         splx(s);
4960                         if ((lun_id != 0) || (device != NULL)) {
4961                                 if (lun_id < (CAM_SCSI2_MAXLUN-1) || phl)
4962                                         lun_id++;
4963                         }
4964                 } else {
4965                         struct cam_ed *device;
4966
4967                         device = request_ccb->ccb_h.path->device;
4968
4969                         if ((device->quirk->quirks & CAM_QUIRK_NOLUNS) == 0) {
4970                                 /* Try the next lun */
4971                                 if (lun_id < (CAM_SCSI2_MAXLUN-1) ||
4972                                     (device->quirk->quirks & CAM_QUIRK_HILUNS))
4973                                         lun_id++;
4974                         }
4975                 }
4976
4977                 xpt_free_path(request_ccb->ccb_h.path);
4978
4979                 /* Check Bounds */
4980                 if ((lun_id == request_ccb->ccb_h.target_lun)
4981                  || lun_id > scan_info->cpi->max_lun) {
4982                         /* We're done */
4983
4984                         xpt_free_ccb(request_ccb);
4985                         scan_info->pending_count--;
4986                         if (scan_info->pending_count == 0) {
4987                                 xpt_free_ccb((union ccb *)scan_info->cpi);
4988                                 request_ccb = scan_info->request_ccb;
4989                                 free(scan_info, M_TEMP);
4990                                 request_ccb->ccb_h.status = CAM_REQ_CMP;
4991                                 xpt_done(request_ccb);
4992                         }
4993                 } else {
4994                         /* Try the next device */
4995                         struct cam_path *path;
4996                         cam_status status;
4997
4998                         path = request_ccb->ccb_h.path;
4999                         status = xpt_create_path(&path, xpt_periph,
5000                                                  path_id, target_id, lun_id);
5001                         if (status != CAM_REQ_CMP) {
5002                                 printf("xpt_scan_bus: xpt_create_path failed "
5003                                        "with status %#x, halting LUN scan\n",
5004                                        status);
5005                                 xpt_free_ccb(request_ccb);
5006                                 scan_info->pending_count--;
5007                                 if (scan_info->pending_count == 0) {
5008                                         xpt_free_ccb(
5009                                                 (union ccb *)scan_info->cpi);
5010                                         request_ccb = scan_info->request_ccb;
5011                                         free(scan_info, M_TEMP);
5012                                         request_ccb->ccb_h.status = CAM_REQ_CMP;
5013                                         xpt_done(request_ccb);
5014                                         break;
5015                                 }
5016                         }
5017                         xpt_setup_ccb(&request_ccb->ccb_h, path,
5018                                       request_ccb->ccb_h.pinfo.priority);
5019                         request_ccb->ccb_h.func_code = XPT_SCAN_LUN;
5020                         request_ccb->ccb_h.cbfcnp = xpt_scan_bus;
5021                         request_ccb->ccb_h.ppriv_ptr0 = scan_info;
5022                         request_ccb->crcn.flags =
5023                                 scan_info->request_ccb->crcn.flags;
5024 #if 0
5025                         xpt_print_path(path);
5026                         printf("xpt_scan bus probing\n");
5027 #endif
5028                         xpt_action(request_ccb);
5029                 }
5030                 break;
5031         }
5032         default:
5033                 break;
5034         }
5035 }
5036
5037 typedef enum {
5038         PROBE_TUR,
5039         PROBE_INQUIRY,
5040         PROBE_FULL_INQUIRY,
5041         PROBE_MODE_SENSE,
5042         PROBE_SERIAL_NUM,
5043         PROBE_TUR_FOR_NEGOTIATION
5044 } probe_action;
5045
5046 typedef enum {
5047         PROBE_INQUIRY_CKSUM     = 0x01,
5048         PROBE_SERIAL_CKSUM      = 0x02,
5049         PROBE_NO_ANNOUNCE       = 0x04
5050 } probe_flags;
5051
5052 typedef struct {
5053         TAILQ_HEAD(, ccb_hdr) request_ccbs;
5054         probe_action    action;
5055         union ccb       saved_ccb;
5056         probe_flags     flags;
5057         MD5_CTX         context;
5058         u_int8_t        digest[16];
5059 } probe_softc;
5060
5061 static void
5062 xpt_scan_lun(struct cam_periph *periph, struct cam_path *path,
5063              cam_flags flags, union ccb *request_ccb)
5064 {
5065         struct ccb_pathinq cpi;
5066         cam_status status;
5067         struct cam_path *new_path;
5068         struct cam_periph *old_periph;
5069         int s;
5070
5071         CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE,
5072                   ("xpt_scan_lun\n"));
5073
5074         xpt_setup_ccb(&cpi.ccb_h, path, /*priority*/1);
5075         cpi.ccb_h.func_code = XPT_PATH_INQ;
5076         xpt_action((union ccb *)&cpi);
5077
5078         if (cpi.ccb_h.status != CAM_REQ_CMP) {
5079                 if (request_ccb != NULL) {
5080                         request_ccb->ccb_h.status = cpi.ccb_h.status;
5081                         xpt_done(request_ccb);
5082                 }
5083                 return;
5084         }
5085
5086         if ((cpi.hba_misc & PIM_NOINITIATOR) != 0) {
5087                 /*
5088                  * Can't scan the bus on an adapter that
5089                  * cannot perform the initiator role.
5090                  */
5091                 if (request_ccb != NULL) {
5092                         request_ccb->ccb_h.status = CAM_REQ_CMP;
5093                         xpt_done(request_ccb);
5094                 }
5095                 return;
5096         }
5097
5098         if (request_ccb == NULL) {
5099                 request_ccb = malloc(sizeof(union ccb), M_TEMP, M_NOWAIT);
5100                 if (request_ccb == NULL) {
5101                         xpt_print_path(path);
5102                         printf("xpt_scan_lun: can't allocate CCB, can't "
5103                                "continue\n");
5104                         return;
5105                 }
5106                 new_path = malloc(sizeof(*new_path), M_TEMP, M_NOWAIT);
5107                 if (new_path == NULL) {
5108                         xpt_print_path(path);
5109                         printf("xpt_scan_lun: can't allocate path, can't "
5110                                "continue\n");
5111                         free(request_ccb, M_TEMP);
5112                         return;
5113                 }
5114                 status = xpt_compile_path(new_path, xpt_periph,
5115                                           path->bus->path_id,
5116                                           path->target->target_id,
5117                                           path->device->lun_id);
5118
5119                 if (status != CAM_REQ_CMP) {
5120                         xpt_print_path(path);
5121                         printf("xpt_scan_lun: can't compile path, can't "
5122                                "continue\n");
5123                         free(request_ccb, M_TEMP);
5124                         free(new_path, M_TEMP);
5125                         return;
5126                 }
5127                 xpt_setup_ccb(&request_ccb->ccb_h, new_path, /*priority*/ 1);
5128                 request_ccb->ccb_h.cbfcnp = xptscandone;
5129                 request_ccb->ccb_h.func_code = XPT_SCAN_LUN;
5130                 request_ccb->crcn.flags = flags;
5131         }
5132
5133         s = splsoftcam();
5134         if ((old_periph = cam_periph_find(path, "probe")) != NULL) {
5135                 probe_softc *softc;
5136
5137                 softc = (probe_softc *)old_periph->softc;
5138                 TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h,
5139                                   periph_links.tqe);
5140         } else {
5141                 status = cam_periph_alloc(proberegister, NULL, probecleanup,
5142                                           probestart, "probe",
5143                                           CAM_PERIPH_BIO,
5144                                           request_ccb->ccb_h.path, NULL, 0,
5145                                           request_ccb);
5146
5147                 if (status != CAM_REQ_CMP) {
5148                         xpt_print_path(path);
5149                         printf("xpt_scan_lun: cam_alloc_periph returned an "
5150                                "error, can't continue probe\n");
5151                         request_ccb->ccb_h.status = status;
5152                         xpt_done(request_ccb);
5153                 }
5154         }
5155         splx(s);
5156 }
5157
5158 static void
5159 xptscandone(struct cam_periph *periph, union ccb *done_ccb)
5160 {
5161         xpt_release_path(done_ccb->ccb_h.path);
5162         free(done_ccb->ccb_h.path, M_TEMP);
5163         free(done_ccb, M_TEMP);
5164 }
5165
5166 static cam_status
5167 proberegister(struct cam_periph *periph, void *arg)
5168 {
5169         union ccb *request_ccb; /* CCB representing the probe request */
5170         probe_softc *softc;
5171
5172         request_ccb = (union ccb *)arg;
5173         if (periph == NULL) {
5174                 printf("proberegister: periph was NULL!!\n");
5175                 return(CAM_REQ_CMP_ERR);
5176         }
5177
5178         if (request_ccb == NULL) {
5179                 printf("proberegister: no probe CCB, "
5180                        "can't register device\n");
5181                 return(CAM_REQ_CMP_ERR);
5182         }
5183
5184         softc = (probe_softc *)malloc(sizeof(*softc), M_TEMP, M_NOWAIT);
5185
5186         if (softc == NULL) {
5187                 printf("proberegister: Unable to probe new device. "
5188                        "Unable to allocate softc\n");
5189                 return(CAM_REQ_CMP_ERR);
5190         }
5191         TAILQ_INIT(&softc->request_ccbs);
5192         TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h,
5193                           periph_links.tqe);
5194         softc->flags = 0;
5195         periph->softc = softc;
5196         cam_periph_acquire(periph);
5197         /*
5198          * Ensure we've waited at least a bus settle
5199          * delay before attempting to probe the device.
5200          * For HBAs that don't do bus resets, this won't make a difference.
5201          */
5202         cam_periph_freeze_after_event(periph, &periph->path->bus->last_reset,
5203                                       SCSI_DELAY);
5204         probeschedule(periph);
5205         return(CAM_REQ_CMP);
5206 }
5207
5208 static void
5209 probeschedule(struct cam_periph *periph)
5210 {
5211         struct ccb_pathinq cpi;
5212         union ccb *ccb;
5213         probe_softc *softc;
5214
5215         softc = (probe_softc *)periph->softc;
5216         ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
5217
5218         xpt_setup_ccb(&cpi.ccb_h, periph->path, /*priority*/1);
5219         cpi.ccb_h.func_code = XPT_PATH_INQ;
5220         xpt_action((union ccb *)&cpi);
5221
5222         /*
5223          * If a device has gone away and another device, or the same one,
5224          * is back in the same place, it should have a unit attention
5225          * condition pending.  It will not report the unit attention in
5226          * response to an inquiry, which may leave invalid transfer
5227          * negotiations in effect.  The TUR will reveal the unit attention
5228          * condition.  Only send the TUR for lun 0, since some devices
5229          * will get confused by commands other than inquiry to non-existent
5230          * luns.  If you think a device has gone away start your scan from
5231          * lun 0.  This will insure that any bogus transfer settings are
5232          * invalidated.
5233          *
5234          * If we haven't seen the device before and the controller supports
5235          * some kind of transfer negotiation, negotiate with the first
5236          * sent command if no bus reset was performed at startup.  This
5237          * ensures that the device is not confused by transfer negotiation
5238          * settings left over by loader or BIOS action.
5239          */
5240         if (((ccb->ccb_h.path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
5241          && (ccb->ccb_h.target_lun == 0)) {
5242                 softc->action = PROBE_TUR;
5243         } else if ((cpi.hba_inquiry & (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE)) != 0
5244               && (cpi.hba_misc & PIM_NOBUSRESET) != 0) {
5245                 proberequestdefaultnegotiation(periph);
5246                 softc->action = PROBE_INQUIRY;
5247         } else {
5248                 softc->action = PROBE_INQUIRY;
5249         }
5250
5251         if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE)
5252                 softc->flags |= PROBE_NO_ANNOUNCE;
5253         else
5254                 softc->flags &= ~PROBE_NO_ANNOUNCE;
5255
5256         xpt_schedule(periph, ccb->ccb_h.pinfo.priority);
5257 }
5258
5259 static void
5260 probestart(struct cam_periph *periph, union ccb *start_ccb)
5261 {
5262         /* Probe the device that our peripheral driver points to */
5263         struct ccb_scsiio *csio;
5264         probe_softc *softc;
5265
5266         CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probestart\n"));
5267
5268         softc = (probe_softc *)periph->softc;
5269         csio = &start_ccb->csio;
5270
5271         switch (softc->action) {
5272         case PROBE_TUR:
5273         case PROBE_TUR_FOR_NEGOTIATION:
5274         {
5275                 scsi_test_unit_ready(csio,
5276                                      /*retries*/4,
5277                                      probedone,
5278                                      MSG_SIMPLE_Q_TAG,
5279                                      SSD_FULL_SIZE,
5280                                      /*timeout*/60000);
5281                 break;
5282         }
5283         case PROBE_INQUIRY:
5284         case PROBE_FULL_INQUIRY:
5285         {
5286                 u_int inquiry_len;
5287                 struct scsi_inquiry_data *inq_buf;
5288
5289                 inq_buf = &periph->path->device->inq_data;
5290                 /*
5291                  * If the device is currently configured, we calculate an
5292                  * MD5 checksum of the inquiry data, and if the serial number
5293                  * length is greater than 0, add the serial number data
5294                  * into the checksum as well.  Once the inquiry and the
5295                  * serial number check finish, we attempt to figure out
5296                  * whether we still have the same device.
5297                  */
5298                 if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
5299
5300                         MD5Init(&softc->context);
5301                         MD5Update(&softc->context, (unsigned char *)inq_buf,
5302                                   sizeof(struct scsi_inquiry_data));
5303                         softc->flags |= PROBE_INQUIRY_CKSUM;
5304                         if (periph->path->device->serial_num_len > 0) {
5305                                 MD5Update(&softc->context,
5306                                           periph->path->device->serial_num,
5307                                           periph->path->device->serial_num_len);
5308                                 softc->flags |= PROBE_SERIAL_CKSUM;
5309                         }
5310                         MD5Final(softc->digest, &softc->context);
5311                 }
5312
5313                 if (softc->action == PROBE_INQUIRY)
5314                         inquiry_len = SHORT_INQUIRY_LENGTH;
5315                 else
5316                         inquiry_len = inq_buf->additional_length + 4;
5317
5318                 scsi_inquiry(csio,
5319                              /*retries*/4,
5320                              probedone,
5321                              MSG_SIMPLE_Q_TAG,
5322                              (u_int8_t *)inq_buf,
5323                              inquiry_len,
5324                              /*evpd*/FALSE,
5325                              /*page_code*/0,
5326                              SSD_MIN_SIZE,
5327                              /*timeout*/60 * 1000);
5328                 break;
5329         }
5330         case PROBE_MODE_SENSE:
5331         {
5332                 void  *mode_buf;
5333                 int    mode_buf_len;
5334
5335                 mode_buf_len = sizeof(struct scsi_mode_header_6)
5336                              + sizeof(struct scsi_mode_blk_desc)
5337                              + sizeof(struct scsi_control_page);
5338                 mode_buf = malloc(mode_buf_len, M_TEMP, M_NOWAIT);
5339                 if (mode_buf != NULL) {
5340                         scsi_mode_sense(csio,
5341                                         /*retries*/4,
5342                                         probedone,
5343                                         MSG_SIMPLE_Q_TAG,
5344                                         /*dbd*/FALSE,
5345                                         SMS_PAGE_CTRL_CURRENT,
5346                                         SMS_CONTROL_MODE_PAGE,
5347                                         mode_buf,
5348                                         mode_buf_len,
5349                                         SSD_FULL_SIZE,
5350                                         /*timeout*/60000);
5351                         break;
5352                 }
5353                 xpt_print_path(periph->path);
5354                 printf("Unable to mode sense control page - malloc failure\n");
5355                 softc->action = PROBE_SERIAL_NUM;
5356                 /* FALLTHROUGH */
5357         }
5358         case PROBE_SERIAL_NUM:
5359         {
5360                 struct scsi_vpd_unit_serial_number *serial_buf;
5361                 struct cam_ed* device;
5362
5363                 serial_buf = NULL;
5364                 device = periph->path->device;
5365                 device->serial_num = NULL;
5366                 device->serial_num_len = 0;
5367
5368                 if ((device->quirk->quirks & CAM_QUIRK_NOSERIAL) == 0)
5369                         serial_buf = (struct scsi_vpd_unit_serial_number *)
5370                                 malloc(sizeof(*serial_buf), M_TEMP, M_NOWAIT);
5371
5372                 if (serial_buf != NULL) {
5373                         bzero(serial_buf, sizeof(*serial_buf));
5374                         scsi_inquiry(csio,
5375                                      /*retries*/4,
5376                                      probedone,
5377                                      MSG_SIMPLE_Q_TAG,
5378                                      (u_int8_t *)serial_buf,
5379                                      sizeof(*serial_buf),
5380                                      /*evpd*/TRUE,
5381                                      SVPD_UNIT_SERIAL_NUMBER,
5382                                      SSD_MIN_SIZE,
5383                                      /*timeout*/60 * 1000);
5384                         break;
5385                 }
5386                 /*
5387                  * We'll have to do without, let our probedone
5388                  * routine finish up for us.
5389                  */
5390                 start_ccb->csio.data_ptr = NULL;
5391                 probedone(periph, start_ccb);
5392                 return;
5393         }
5394         }
5395         xpt_action(start_ccb);
5396 }
5397
5398 static void
5399 proberequestdefaultnegotiation(struct cam_periph *periph)
5400 {
5401         struct ccb_trans_settings cts;
5402
5403         xpt_setup_ccb(&cts.ccb_h, periph->path, /*priority*/1);
5404         cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
5405         cts.flags = CCB_TRANS_USER_SETTINGS;
5406         xpt_action((union ccb *)&cts);
5407         cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
5408         cts.flags &= ~CCB_TRANS_USER_SETTINGS;
5409         cts.flags |= CCB_TRANS_CURRENT_SETTINGS;
5410         xpt_action((union ccb *)&cts);
5411 }
5412
5413 static void
5414 probedone(struct cam_periph *periph, union ccb *done_ccb)
5415 {
5416         probe_softc *softc;
5417         struct cam_path *path;
5418         u_int32_t  priority;
5419
5420         CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probedone\n"));
5421
5422         softc = (probe_softc *)periph->softc;
5423         path = done_ccb->ccb_h.path;
5424         priority = done_ccb->ccb_h.pinfo.priority;
5425
5426         switch (softc->action) {
5427         case PROBE_TUR:
5428         {
5429                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
5430
5431                         if (cam_periph_error(done_ccb, 0,
5432                                              SF_NO_PRINT, NULL) == ERESTART)
5433                                 return;
5434                         else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
5435                                 /* Don't wedge the queue */
5436                                 xpt_release_devq(done_ccb->ccb_h.path,
5437                                                  /*count*/1,
5438                                                  /*run_queue*/TRUE);
5439                 }
5440                 softc->action = PROBE_INQUIRY;
5441                 xpt_release_ccb(done_ccb);
5442                 xpt_schedule(periph, priority);
5443                 return;
5444         }
5445         case PROBE_INQUIRY:
5446         case PROBE_FULL_INQUIRY:
5447         {
5448                 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5449                         struct scsi_inquiry_data *inq_buf;
5450                         u_int8_t periph_qual;
5451                         u_int8_t periph_dtype;
5452
5453                         path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;
5454                         inq_buf = &path->device->inq_data;
5455
5456                         periph_qual = SID_QUAL(inq_buf);
5457                         periph_dtype = SID_TYPE(inq_buf);
5458
5459                         if (periph_dtype != T_NODEVICE) {
5460                                 switch(periph_qual) {
5461                                 case SID_QUAL_LU_CONNECTED:
5462                                 {
5463                                         u_int8_t alen;
5464
5465                                         /*
5466                                          * We conservatively request only
5467                                          * SHORT_INQUIRY_LEN bytes of inquiry
5468                                          * information during our first try
5469                                          * at sending an INQUIRY. If the device
5470                                          * has more information to give,
5471                                          * perform a second request specifying
5472                                          * the amount of information the device
5473                                          * is willing to give.
5474                                          */
5475                                         alen = inq_buf->additional_length;
5476                                         if (softc->action == PROBE_INQUIRY
5477                                          && alen > (SHORT_INQUIRY_LENGTH - 4)) {
5478                                                 softc->action =
5479                                                     PROBE_FULL_INQUIRY;
5480                                                 xpt_release_ccb(done_ccb);
5481                                                 xpt_schedule(periph, priority);
5482                                                 return;
5483                                         }
5484
5485                                         xpt_find_quirk(path->device);
5486
5487                                         if ((inq_buf->flags & SID_CmdQue) != 0)
5488                                                 softc->action =
5489                                                     PROBE_MODE_SENSE;
5490                                         else
5491                                                 softc->action =
5492                                                     PROBE_SERIAL_NUM;
5493
5494                                         path->device->flags &=
5495                                                 ~CAM_DEV_UNCONFIGURED;
5496
5497                                         xpt_release_ccb(done_ccb);
5498                                         xpt_schedule(periph, priority);
5499                                         return;
5500                                 }
5501                                 default:
5502                                         break;
5503                                 }
5504                         }
5505                 } else if (cam_periph_error(done_ccb, 0,
5506                                             done_ccb->ccb_h.target_lun > 0
5507                                             ? SF_RETRY_UA|SF_QUIET_IR
5508                                             : SF_RETRY_UA,
5509                                             &softc->saved_ccb) == ERESTART) {
5510                         return;
5511                 } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5512                         /* Don't wedge the queue */
5513                         xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
5514                                          /*run_queue*/TRUE);
5515                 }
5516                 /*
5517                  * If we get to this point, we got an error status back
5518                  * from the inquiry and the error status doesn't require
5519                  * automatically retrying the command.  Therefore, the
5520                  * inquiry failed.  If we had inquiry information before
5521                  * for this device, but this latest inquiry command failed,
5522                  * the device has probably gone away.  If this device isn't
5523                  * already marked unconfigured, notify the peripheral
5524                  * drivers that this device is no more.
5525                  */
5526                 if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
5527                         /* Send the async notification. */
5528                         xpt_async(AC_LOST_DEVICE, path, NULL);
5529
5530                 xpt_release_ccb(done_ccb);
5531                 break;
5532         }
5533         case PROBE_MODE_SENSE:
5534         {
5535                 struct ccb_scsiio *csio;
5536                 struct scsi_mode_header_6 *mode_hdr;
5537
5538                 csio = &done_ccb->csio;
5539                 mode_hdr = (struct scsi_mode_header_6 *)csio->data_ptr;
5540                 if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
5541                         struct scsi_control_page *page;
5542                         u_int8_t *offset;
5543
5544                         offset = ((u_int8_t *)&mode_hdr[1])
5545                             + mode_hdr->blk_desc_len;
5546                         page = (struct scsi_control_page *)offset;
5547                         path->device->queue_flags = page->queue_flags;
5548                 } else if (cam_periph_error(done_ccb, 0,
5549                                             SF_RETRY_UA|SF_NO_PRINT,
5550                                             &softc->saved_ccb) == ERESTART) {
5551                         return;
5552                 } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5553                         /* Don't wedge the queue */
5554                         xpt_release_devq(done_ccb->ccb_h.path,
5555                                          /*count*/1, /*run_queue*/TRUE);
5556                 }
5557                 xpt_release_ccb(done_ccb);
5558                 free(mode_hdr, M_TEMP);
5559                 softc->action = PROBE_SERIAL_NUM;
5560                 xpt_schedule(periph, priority);
5561                 return;
5562         }
5563         case PROBE_SERIAL_NUM:
5564         {
5565                 struct ccb_scsiio *csio;
5566                 struct scsi_vpd_unit_serial_number *serial_buf;
5567                 u_int32_t  priority;
5568                 int changed;
5569                 int have_serialnum;
5570
5571                 changed = 1;
5572                 have_serialnum = 0;
5573                 csio = &done_ccb->csio;
5574                 priority = done_ccb->ccb_h.pinfo.priority;
5575                 serial_buf =
5576                     (struct scsi_vpd_unit_serial_number *)csio->data_ptr;
5577
5578                 /* Clean up from previous instance of this device */
5579                 if (path->device->serial_num != NULL) {
5580                         free(path->device->serial_num, M_DEVBUF);
5581                         path->device->serial_num = NULL;
5582                         path->device->serial_num_len = 0;
5583                 }
5584
5585                 if (serial_buf == NULL) {
5586                         /*
5587                          * Don't process the command as it was never sent
5588                          */
5589                 } else if ((csio->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP
5590                         && (serial_buf->length > 0)) {
5591
5592                         have_serialnum = 1;
5593                         path->device->serial_num =
5594                                 (u_int8_t *)malloc((serial_buf->length + 1),
5595                                                    M_DEVBUF, M_NOWAIT);
5596                         if (path->device->serial_num != NULL) {
5597                                 bcopy(serial_buf->serial_num,
5598                                       path->device->serial_num,
5599                                       serial_buf->length);
5600                                 path->device->serial_num_len =
5601                                     serial_buf->length;
5602                                 path->device->serial_num[serial_buf->length]
5603                                     = '\0';
5604                         }
5605                 } else if (cam_periph_error(done_ccb, 0,
5606                                             SF_RETRY_UA|SF_NO_PRINT,
5607                                             &softc->saved_ccb) == ERESTART) {
5608                         return;
5609                 } else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5610                         /* Don't wedge the queue */
5611                         xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
5612                                          /*run_queue*/TRUE);
5613                 }
5614
5615                 /*
5616                  * Let's see if we have seen this device before.
5617                  */
5618                 if ((softc->flags & PROBE_INQUIRY_CKSUM) != 0) {
5619                         MD5_CTX context;
5620                         u_int8_t digest[16];
5621
5622                         MD5Init(&context);
5623
5624                         MD5Update(&context,
5625                                   (unsigned char *)&path->device->inq_data,
5626                                   sizeof(struct scsi_inquiry_data));
5627
5628                         if (have_serialnum)
5629                                 MD5Update(&context, serial_buf->serial_num,
5630                                           serial_buf->length);
5631
5632                         MD5Final(digest, &context);
5633                         if (bcmp(softc->digest, digest, 16) == 0)
5634                                 changed = 0;
5635
5636                         /*
5637                          * XXX Do we need to do a TUR in order to ensure
5638                          *     that the device really hasn't changed???
5639                          */
5640                         if ((changed != 0)
5641                          && ((softc->flags & PROBE_NO_ANNOUNCE) == 0))
5642                                 xpt_async(AC_LOST_DEVICE, path, NULL);
5643                 }
5644                 if (serial_buf != NULL)
5645                         free(serial_buf, M_TEMP);
5646
5647                 if (changed != 0) {
5648                         /*
5649                          * Now that we have all the necessary
5650                          * information to safely perform transfer
5651                          * negotiations... Controllers don't perform
5652                          * any negotiation or tagged queuing until
5653                          * after the first XPT_SET_TRAN_SETTINGS ccb is
5654                          * received.  So, on a new device, just retreive
5655                          * the user settings, and set them as the current
5656                          * settings to set the device up.
5657                          */
5658                         proberequestdefaultnegotiation(periph);
5659                         xpt_release_ccb(done_ccb);
5660
5661                         /*
5662                          * Perform a TUR to allow the controller to
5663                          * perform any necessary transfer negotiation.
5664                          */
5665                         softc->action = PROBE_TUR_FOR_NEGOTIATION;
5666                         xpt_schedule(periph, priority);
5667                         return;
5668                 }
5669                 xpt_release_ccb(done_ccb);
5670                 break;
5671         }
5672         case PROBE_TUR_FOR_NEGOTIATION:
5673                 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
5674                         /* Don't wedge the queue */
5675                         xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
5676                                          /*run_queue*/TRUE);
5677                 }
5678
5679                 path->device->flags &= ~CAM_DEV_UNCONFIGURED;
5680
5681                 if ((softc->flags & PROBE_NO_ANNOUNCE) == 0) {
5682                         /* Inform the XPT that a new device has been found */
5683                         done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
5684                         xpt_action(done_ccb);
5685
5686                         xpt_async(AC_FOUND_DEVICE, xpt_periph->path, done_ccb);
5687                 }
5688                 xpt_release_ccb(done_ccb);
5689                 break;
5690         }
5691         done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
5692         TAILQ_REMOVE(&softc->request_ccbs, &done_ccb->ccb_h, periph_links.tqe);
5693         done_ccb->ccb_h.status = CAM_REQ_CMP;
5694         xpt_done(done_ccb);
5695         if (TAILQ_FIRST(&softc->request_ccbs) == NULL) {
5696                 cam_periph_invalidate(periph);
5697                 cam_periph_release(periph);
5698         } else {
5699                 probeschedule(periph);
5700         }
5701 }
5702
5703 static void
5704 probecleanup(struct cam_periph *periph)
5705 {
5706         free(periph->softc, M_TEMP);
5707 }
5708
5709 static void
5710 xpt_find_quirk(struct cam_ed *device)
5711 {
5712         caddr_t match;
5713
5714         match = cam_quirkmatch((caddr_t)&device->inq_data,
5715                                (caddr_t)xpt_quirk_table,
5716                                sizeof(xpt_quirk_table)/sizeof(*xpt_quirk_table),
5717                                sizeof(*xpt_quirk_table), scsi_inquiry_match);
5718
5719         if (match == NULL)
5720                 panic("xpt_find_quirk: device didn't match wildcard entry!!");
5721
5722         device->quirk = (struct xpt_quirk_entry *)match;
5723 }
5724
5725 static void
5726 xpt_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_ed *device,
5727                           int async_update)
5728 {
5729         struct  cam_sim *sim;
5730         int     qfrozen;
5731
5732         sim = cts->ccb_h.path->bus->sim;
5733         if (async_update == FALSE) {
5734                 struct  scsi_inquiry_data *inq_data;
5735                 struct  ccb_pathinq cpi;
5736                 struct  ccb_trans_settings cur_cts;
5737
5738                 if (device == NULL) {
5739                         cts->ccb_h.status = CAM_PATH_INVALID;
5740                         xpt_done((union ccb *)cts);
5741                         return;
5742                 }
5743
5744                 /*
5745                  * Perform sanity checking against what the
5746                  * controller and device can do.
5747                  */
5748                 xpt_setup_ccb(&cpi.ccb_h, cts->ccb_h.path, /*priority*/1);
5749                 cpi.ccb_h.func_code = XPT_PATH_INQ;
5750                 xpt_action((union ccb *)&cpi);
5751                 xpt_setup_ccb(&cur_cts.ccb_h, cts->ccb_h.path, /*priority*/1);
5752                 cur_cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
5753                 cur_cts.flags = CCB_TRANS_CURRENT_SETTINGS;
5754                 xpt_action((union ccb *)&cur_cts);
5755                 inq_data = &device->inq_data;
5756
5757                 /* Fill in any gaps in what the user gave us */
5758                 if ((cts->valid & CCB_TRANS_SYNC_RATE_VALID) == 0)
5759                         cts->sync_period = cur_cts.sync_period;
5760                 if ((cts->valid & CCB_TRANS_SYNC_OFFSET_VALID) == 0)
5761                         cts->sync_offset = cur_cts.sync_offset;
5762                 if ((cts->valid & CCB_TRANS_BUS_WIDTH_VALID) == 0)
5763                         cts->bus_width = cur_cts.bus_width;
5764                 if ((cts->valid & CCB_TRANS_DISC_VALID) == 0) {
5765                         cts->flags &= ~CCB_TRANS_DISC_ENB;
5766                         cts->flags |= cur_cts.flags & CCB_TRANS_DISC_ENB;
5767                 }
5768                 if ((cts->valid & CCB_TRANS_TQ_VALID) == 0) {
5769                         cts->flags &= ~CCB_TRANS_TAG_ENB;
5770                         cts->flags |= cur_cts.flags & CCB_TRANS_TAG_ENB;
5771                 }
5772                 if (((device->flags & CAM_DEV_INQUIRY_DATA_VALID) != 0
5773                   && (inq_data->flags & SID_Sync) == 0)
5774                  || (cpi.hba_inquiry & PI_SDTR_ABLE) == 0) {
5775                         /* Force async */
5776                         cts->sync_period = 0;
5777                         cts->sync_offset = 0;
5778                 }
5779
5780                 switch (cts->bus_width) {
5781                 case MSG_EXT_WDTR_BUS_32_BIT:
5782                         if (((device->flags & CAM_DEV_INQUIRY_DATA_VALID) == 0
5783                           || (inq_data->flags & SID_WBus32) != 0)
5784                          && (cpi.hba_inquiry & PI_WIDE_32) != 0)
5785                                 break;
5786                         /* Fall Through to 16-bit */
5787                 case MSG_EXT_WDTR_BUS_16_BIT:
5788                         if (((device->flags & CAM_DEV_INQUIRY_DATA_VALID) == 0
5789                           || (inq_data->flags & SID_WBus16) != 0)
5790                          && (cpi.hba_inquiry & PI_WIDE_16) != 0) {
5791                                 cts->bus_width = MSG_EXT_WDTR_BUS_16_BIT;
5792                                 break;
5793                         }
5794                         /* Fall Through to 8-bit */
5795                 default: /* New bus width?? */
5796                 case MSG_EXT_WDTR_BUS_8_BIT:
5797                         /* All targets can do this */
5798                         cts->bus_width = MSG_EXT_WDTR_BUS_8_BIT;
5799                         break;
5800                 }
5801
5802                 if ((cts->flags & CCB_TRANS_DISC_ENB) == 0) {
5803                         /*
5804                          * Can't tag queue without disconnection.
5805                          */
5806                         cts->flags &= ~CCB_TRANS_TAG_ENB;
5807                         cts->valid |= CCB_TRANS_TQ_VALID;
5808                 }
5809
5810                 if ((cpi.hba_inquiry & PI_TAG_ABLE) == 0
5811                  || (inq_data->flags & SID_CmdQue) == 0
5812                  || (device->queue_flags & SCP_QUEUE_DQUE) != 0
5813                  || (device->quirk->mintags == 0)) {
5814                         /*
5815                          * Can't tag on hardware that doesn't support,
5816                          * doesn't have it enabled, or has broken tag support.
5817                          */
5818                         cts->flags &= ~CCB_TRANS_TAG_ENB;
5819                 }
5820         }
5821
5822         qfrozen = FALSE;
5823         if ((cts->valid & CCB_TRANS_TQ_VALID) != 0
5824          && (async_update == FALSE)) {
5825                 int device_tagenb;
5826
5827                 /*
5828                  * If we are transitioning from tags to no-tags or
5829                  * vice-versa, we need to carefully freeze and restart
5830                  * the queue so that we don't overlap tagged and non-tagged
5831                  * commands.  We also temporarily stop tags if there is
5832                  * a change in transfer negotiation settings to allow
5833                  * "tag-less" negotiation.
5834                  */
5835                 if ((device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5836                  || (device->inq_flags & SID_CmdQue) != 0)
5837                         device_tagenb = TRUE;
5838                 else
5839                         device_tagenb = FALSE;
5840
5841                 if (((cts->flags & CCB_TRANS_TAG_ENB) != 0
5842                   && device_tagenb == FALSE)
5843                  || ((cts->flags & CCB_TRANS_TAG_ENB) == 0
5844                   && device_tagenb == TRUE)) {
5845
5846                         if ((cts->flags & CCB_TRANS_TAG_ENB) != 0) {
5847                                 /*
5848                                  * Delay change to use tags until after a
5849                                  * few commands have gone to this device so
5850                                  * the controller has time to perform transfer
5851                                  * negotiations without tagged messages getting
5852                                  * in the way.
5853                                  */
5854                                 device->tag_delay_count = CAM_TAG_DELAY_COUNT;
5855                                 device->flags |= CAM_DEV_TAG_AFTER_COUNT;
5856                         } else {
5857                                 xpt_freeze_devq(cts->ccb_h.path, /*count*/1);
5858                                 qfrozen = TRUE;
5859                                 device->inq_flags &= ~SID_CmdQue;
5860                                 xpt_dev_ccbq_resize(cts->ccb_h.path,
5861                                                     sim->max_dev_openings);
5862                                 device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
5863                                 device->tag_delay_count = 0;
5864                         }
5865                 }
5866         }
5867
5868         if (async_update == FALSE) {
5869                 /*
5870                  * If we are currently performing tagged transactions to
5871                  * this device and want to change its negotiation parameters,
5872                  * go non-tagged for a bit to give the controller a chance to
5873                  * negotiate unhampered by tag messages.
5874                  */
5875                 if ((device->inq_flags & SID_CmdQue) != 0
5876                  && (cts->flags & (CCB_TRANS_SYNC_RATE_VALID|
5877                                    CCB_TRANS_SYNC_OFFSET_VALID|
5878                                    CCB_TRANS_BUS_WIDTH_VALID)) != 0)
5879                         xpt_toggle_tags(cts->ccb_h.path);
5880
5881                 (*(sim->sim_action))(sim, (union ccb *)cts);
5882         }
5883
5884         if (qfrozen) {
5885                 struct ccb_relsim crs;
5886
5887                 xpt_setup_ccb(&crs.ccb_h, cts->ccb_h.path,
5888                               /*priority*/1);
5889                 crs.ccb_h.func_code = XPT_REL_SIMQ;
5890                 crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
5891                 crs.openings
5892                     = crs.release_timeout
5893                     = crs.qfrozen_cnt
5894                     = 0;
5895                 xpt_action((union ccb *)&crs);
5896         }
5897 }
5898
5899 static void
5900 xpt_toggle_tags(struct cam_path *path)
5901 {
5902         struct cam_ed *dev;
5903
5904         /*
5905          * Give controllers a chance to renegotiate
5906          * before starting tag operations.  We
5907          * "toggle" tagged queuing off then on
5908          * which causes the tag enable command delay
5909          * counter to come into effect.
5910          */
5911         dev = path->device;
5912         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
5913          || ((dev->inq_flags & SID_CmdQue) != 0
5914           && (dev->inq_flags & (SID_Sync|SID_WBus16|SID_WBus32)) != 0)) {
5915                 struct ccb_trans_settings cts;
5916
5917                 xpt_setup_ccb(&cts.ccb_h, path, 1);
5918                 cts.flags = 0;
5919                 cts.valid = CCB_TRANS_TQ_VALID;
5920                 xpt_set_transfer_settings(&cts, path->device,
5921                                           /*async_update*/TRUE);
5922                 cts.flags = CCB_TRANS_TAG_ENB;
5923                 xpt_set_transfer_settings(&cts, path->device,
5924                                           /*async_update*/TRUE);
5925         }
5926 }
5927
5928 static void
5929 xpt_start_tags(struct cam_path *path)
5930 {
5931         struct ccb_relsim crs;
5932         struct cam_ed *device;
5933         struct cam_sim *sim;
5934         int    newopenings;
5935
5936         device = path->device;
5937         sim = path->bus->sim;
5938         device->flags &= ~CAM_DEV_TAG_AFTER_COUNT;
5939         xpt_freeze_devq(path, /*count*/1);
5940         device->inq_flags |= SID_CmdQue;
5941         newopenings = min(device->quirk->maxtags, sim->max_tagged_dev_openings);
5942         xpt_dev_ccbq_resize(path, newopenings);
5943         xpt_setup_ccb(&crs.ccb_h, path, /*priority*/1);
5944         crs.ccb_h.func_code = XPT_REL_SIMQ;
5945         crs.release_flags = RELSIM_RELEASE_AFTER_QEMPTY;
5946         crs.openings
5947             = crs.release_timeout
5948             = crs.qfrozen_cnt
5949             = 0;
5950         xpt_action((union ccb *)&crs);
5951 }
5952
5953 static int busses_to_config;
5954 static int busses_to_reset;
5955
5956 static int
5957 xptconfigbuscountfunc(struct cam_eb *bus, void *arg)
5958 {
5959         if (bus->path_id != CAM_XPT_PATH_ID) {
5960                 struct cam_path path;
5961                 struct ccb_pathinq cpi;
5962                 int can_negotiate;
5963
5964                 busses_to_config++;
5965                 xpt_compile_path(&path, NULL, bus->path_id,
5966                                  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
5967                 xpt_setup_ccb(&cpi.ccb_h, &path, /*priority*/1);
5968                 cpi.ccb_h.func_code = XPT_PATH_INQ;
5969                 xpt_action((union ccb *)&cpi);
5970                 can_negotiate = cpi.hba_inquiry;
5971                 can_negotiate &= (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE);
5972                 if ((cpi.hba_misc & PIM_NOBUSRESET) == 0
5973                  && can_negotiate)
5974                         busses_to_reset++;
5975                 xpt_release_path(&path);
5976         }
5977
5978         return(1);
5979 }
5980
5981 static int
5982 xptconfigfunc(struct cam_eb *bus, void *arg)
5983 {
5984         struct  cam_path *path;
5985         union   ccb *work_ccb;
5986
5987         if (bus->path_id != CAM_XPT_PATH_ID) {
5988                 cam_status status;
5989                 int can_negotiate;
5990
5991                 work_ccb = xpt_alloc_ccb();
5992                 if ((status = xpt_create_path(&path, xpt_periph, bus->path_id,
5993                                               CAM_TARGET_WILDCARD,
5994                                               CAM_LUN_WILDCARD)) !=CAM_REQ_CMP){
5995                         printf("xptconfigfunc: xpt_create_path failed with "
5996                                "status %#x for bus %d\n", status, bus->path_id);
5997                         printf("xptconfigfunc: halting bus configuration\n");
5998                         xpt_free_ccb(work_ccb);
5999                         busses_to_config--;
6000                         xpt_finishconfig(xpt_periph, NULL);
6001                         return(0);
6002                 }
6003                 xpt_setup_ccb(&work_ccb->ccb_h, path, /*priority*/1);
6004                 work_ccb->ccb_h.func_code = XPT_PATH_INQ;
6005                 xpt_action(work_ccb);
6006                 if (work_ccb->ccb_h.status != CAM_REQ_CMP) {
6007                         printf("xptconfigfunc: CPI failed on bus %d "
6008                                "with status %d\n", bus->path_id,
6009                                work_ccb->ccb_h.status);
6010                         xpt_finishconfig(xpt_periph, work_ccb);
6011                         return(1);
6012                 }
6013
6014                 can_negotiate = work_ccb->cpi.hba_inquiry;
6015                 can_negotiate &= (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE);
6016                 if ((work_ccb->cpi.hba_misc & PIM_NOBUSRESET) == 0
6017                  && (can_negotiate != 0)) {
6018                         xpt_setup_ccb(&work_ccb->ccb_h, path, /*priority*/1);
6019                         work_ccb->ccb_h.func_code = XPT_RESET_BUS;
6020                         work_ccb->ccb_h.cbfcnp = NULL;
6021                         CAM_DEBUG(path, CAM_DEBUG_SUBTRACE,
6022                                   ("Resetting Bus\n"));
6023                         xpt_action(work_ccb);
6024                         xpt_finishconfig(xpt_periph, work_ccb);
6025                 } else {
6026                         /* Act as though we performed a successful BUS RESET */
6027                         work_ccb->ccb_h.func_code = XPT_RESET_BUS;
6028                         xpt_finishconfig(xpt_periph, work_ccb);
6029                 }
6030         }
6031
6032         return(1);
6033 }
6034
6035 static void
6036 xpt_config(void *arg)
6037 {
6038         /* Now that interrupts are enabled, go find our devices */
6039
6040 #ifdef CAMDEBUG
6041         /* Setup debugging flags and path */
6042 #ifdef CAM_DEBUG_FLAGS
6043         cam_dflags = CAM_DEBUG_FLAGS;
6044 #else /* !CAM_DEBUG_FLAGS */
6045         cam_dflags = CAM_DEBUG_NONE;
6046 #endif /* CAM_DEBUG_FLAGS */
6047 #ifdef CAM_DEBUG_BUS
6048         if (cam_dflags != CAM_DEBUG_NONE) {
6049                 if (xpt_create_path(&cam_dpath, xpt_periph,
6050                                     CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
6051                                     CAM_DEBUG_LUN) != CAM_REQ_CMP) {
6052                         printf("xpt_config: xpt_create_path() failed for debug"
6053                                " target %d:%d:%d, debugging disabled\n",
6054                                CAM_DEBUG_BUS, CAM_DEBUG_TARGET, CAM_DEBUG_LUN);
6055                         cam_dflags = CAM_DEBUG_NONE;
6056                 }
6057         } else
6058                 cam_dpath = NULL;
6059 #else /* !CAM_DEBUG_BUS */
6060         cam_dpath = NULL;
6061 #endif /* CAM_DEBUG_BUS */
6062 #endif /* CAMDEBUG */
6063
6064         /*
6065          * Scan all installed busses.
6066          */
6067         xpt_for_all_busses(xptconfigbuscountfunc, NULL);
6068
6069         if (busses_to_config == 0) {
6070                 /* Call manually because we don't have any busses */
6071                 xpt_finishconfig(xpt_periph, NULL);
6072         } else  {
6073                 if (busses_to_reset > 0 && SCSI_DELAY >= 2000) {
6074                         printf("Waiting %d seconds for SCSI "
6075                                "devices to settle\n", SCSI_DELAY/1000);
6076                 }
6077                 xpt_for_all_busses(xptconfigfunc, NULL);
6078         }
6079 }
6080
6081 /*
6082  * If the given device only has one peripheral attached to it, and if that
6083  * peripheral is the passthrough driver, announce it.  This insures that the
6084  * user sees some sort of announcement for every peripheral in their system.
6085  */
6086 static int
6087 xptpassannouncefunc(struct cam_ed *device, void *arg)
6088 {
6089         struct cam_periph *periph;
6090         int i;
6091
6092         for (periph = SLIST_FIRST(&device->periphs), i = 0; periph != NULL;
6093              periph = SLIST_NEXT(periph, periph_links), i++);
6094
6095         periph = SLIST_FIRST(&device->periphs);
6096         if ((i == 1)
6097          && (strncmp(periph->periph_name, "pass", 4) == 0))
6098                 xpt_announce_periph(periph, NULL);
6099
6100         return(1);
6101 }
6102
6103 static void
6104 xpt_finishconfig(struct cam_periph *periph, union ccb *done_ccb)
6105 {
6106         struct  periph_driver **p_drv;
6107         int     i;
6108
6109         if (done_ccb != NULL) {
6110                 CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
6111                           ("xpt_finishconfig\n"));
6112                 switch(done_ccb->ccb_h.func_code) {
6113                 case XPT_RESET_BUS:
6114                         if (done_ccb->ccb_h.status == CAM_REQ_CMP) {
6115                                 done_ccb->ccb_h.func_code = XPT_SCAN_BUS;
6116                                 done_ccb->ccb_h.cbfcnp = xpt_finishconfig;
6117                                 xpt_action(done_ccb);
6118                                 return;
6119                         }
6120                         /* FALLTHROUGH */
6121                 case XPT_SCAN_BUS:
6122                 default:
6123                         xpt_free_path(done_ccb->ccb_h.path);
6124                         busses_to_config--;
6125                         break;
6126                 }
6127         }
6128
6129         if (busses_to_config == 0) {
6130                 /* Register all the peripheral drivers */
6131                 /* XXX This will have to change when we have loadable modules */
6132                 p_drv = (struct periph_driver **)periphdriver_set.ls_items;
6133                 for (i = 0; p_drv[i] != NULL; i++) {
6134                         (*p_drv[i]->init)();
6135                 }
6136
6137                 /*
6138                  * Check for devices with no "standard" peripheral driver
6139                  * attached.  For any devices like that, announce the
6140                  * passthrough driver so the user will see something.
6141                  */
6142                 xpt_for_all_devices(xptpassannouncefunc, NULL);
6143
6144                 /* Release our hook so that the boot can continue. */
6145                 config_intrhook_disestablish(xpt_config_hook);
6146                 free(xpt_config_hook, M_TEMP);
6147                 xpt_config_hook = NULL;
6148         }
6149         if (done_ccb != NULL)
6150                 xpt_free_ccb(done_ccb);
6151 }
6152
6153 static void
6154 xptaction(struct cam_sim *sim, union ccb *work_ccb)
6155 {
6156         CAM_DEBUG(work_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xptaction\n"));
6157
6158         switch (work_ccb->ccb_h.func_code) {
6159         /* Common cases first */
6160         case XPT_PATH_INQ:              /* Path routing inquiry */
6161         {
6162                 struct ccb_pathinq *cpi;
6163
6164                 cpi = &work_ccb->cpi;
6165                 cpi->version_num = 1; /* XXX??? */
6166                 cpi->hba_inquiry = 0;
6167                 cpi->target_sprt = 0;
6168                 cpi->hba_misc = 0;
6169                 cpi->hba_eng_cnt = 0;
6170                 cpi->max_target = 0;
6171                 cpi->max_lun = 0;
6172                 cpi->initiator_id = 0;
6173                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
6174                 strncpy(cpi->hba_vid, "", HBA_IDLEN);
6175                 strncpy(cpi->dev_name, sim->sim_name, DEV_IDLEN);
6176                 cpi->unit_number = sim->unit_number;
6177                 cpi->bus_id = sim->bus_id;
6178                 cpi->base_transfer_speed = 0;
6179                 cpi->ccb_h.status = CAM_REQ_CMP;
6180                 xpt_done(work_ccb);
6181                 break;
6182         }
6183         default:
6184                 work_ccb->ccb_h.status = CAM_REQ_INVALID;
6185                 xpt_done(work_ccb);
6186                 break;
6187         }
6188 }
6189
6190 /*
6191  * The xpt as a "controller" has no interrupt sources, so polling
6192  * is a no-op.
6193  */
6194 static void
6195 xptpoll(struct cam_sim *sim)
6196 {
6197 }
6198
6199 /*
6200  * Should only be called by the machine interrupt dispatch routines,
6201  * so put these prototypes here instead of in the header.
6202  */
6203
6204 static void
6205 swi_camnet(void)
6206 {
6207         camisr(&cam_netq);
6208 }
6209
6210 static void
6211 swi_cambio(void)
6212 {
6213         camisr(&cam_bioq);
6214 }
6215
6216 static void
6217 camisr(cam_isrq_t *queue)
6218 {
6219         int     s;
6220         struct  ccb_hdr *ccb_h;
6221
6222         s = splcam();
6223         while ((ccb_h = TAILQ_FIRST(queue)) != NULL) {
6224                 int     runq;
6225
6226                 TAILQ_REMOVE(queue, ccb_h, sim_links.tqe);
6227                 ccb_h->pinfo.index = CAM_UNQUEUED_INDEX;
6228                 splx(s);
6229
6230                 CAM_DEBUG(ccb_h->path, CAM_DEBUG_TRACE,
6231                           ("camisr"));
6232
6233                 runq = FALSE;
6234
6235                 if (ccb_h->flags & CAM_HIGH_POWER) {
6236                         struct highpowerlist    *hphead;
6237                         struct cam_ed           *device;
6238                         union ccb               *send_ccb;
6239
6240                         hphead = &highpowerq;
6241
6242                         send_ccb = (union ccb *)STAILQ_FIRST(hphead);
6243
6244                         /*
6245                          * Increment the count since this command is done.
6246                          */
6247                         num_highpower++;
6248
6249                         /*
6250                          * Any high powered commands queued up?
6251                          */
6252                         if (send_ccb != NULL) {
6253                                 device = send_ccb->ccb_h.path->device;
6254
6255                                 STAILQ_REMOVE_HEAD(hphead, xpt_links.stqe);
6256
6257                                 xpt_release_devq(send_ccb->ccb_h.path,
6258                                                  /*count*/1, /*runqueue*/TRUE);
6259                         }
6260                 }
6261                 if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
6262                         struct cam_ed *dev;
6263
6264                         dev = ccb_h->path->device;
6265
6266                         s = splcam();
6267                         cam_ccbq_ccb_done(&dev->ccbq, (union ccb *)ccb_h);
6268
6269                         ccb_h->path->bus->sim->devq->send_active--;
6270                         ccb_h->path->bus->sim->devq->send_openings++;
6271                         splx(s);
6272
6273                         if ((dev->flags & CAM_DEV_REL_ON_COMPLETE) != 0
6274                          || ((dev->flags & CAM_DEV_REL_ON_QUEUE_EMPTY) != 0
6275                           && (dev->ccbq.dev_active == 0))) {
6276
6277                                 xpt_release_devq(ccb_h->path, /*count*/1,
6278                                                  /*run_queue*/TRUE);
6279                         }
6280
6281                         if ((dev->flags & CAM_DEV_TAG_AFTER_COUNT) != 0
6282                          && (--dev->tag_delay_count == 0))
6283                                 xpt_start_tags(ccb_h->path);
6284
6285                         if ((dev->ccbq.queue.entries > 0)
6286                          && (dev->qfrozen_cnt == 0)
6287                          && (device_is_send_queued(dev) == 0)) {
6288                                 runq = xpt_schedule_dev_sendq(ccb_h->path->bus,
6289                                                               dev);
6290                         }
6291                 }
6292
6293                 if (ccb_h->status & CAM_RELEASE_SIMQ) {
6294                         xpt_release_simq(ccb_h->path->bus->sim,
6295                                          /*run_queue*/TRUE);
6296                         ccb_h->status &= ~CAM_RELEASE_SIMQ;
6297                         runq = FALSE;
6298                 }
6299
6300                 if ((ccb_h->flags & CAM_DEV_QFRZDIS)
6301                  && (ccb_h->status & CAM_DEV_QFRZN)) {
6302                         xpt_release_devq(ccb_h->path, /*count*/1,
6303                                          /*run_queue*/TRUE);
6304                         ccb_h->status &= ~CAM_DEV_QFRZN;
6305                 } else if (runq) {
6306                         xpt_run_dev_sendq(ccb_h->path->bus);
6307                 }
6308
6309                 /* Call the peripheral driver's callback */
6310                 (*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
6311
6312                 /* Raise IPL for while test */
6313                 s = splcam();
6314         }
6315         splx(s);
6316 }