sys/dev/isp/isp_freebsd.c

   1 /*-
   2  * Copyright (c) 1997-2009 by Matthew Jacob
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice immediately at the beginning of the file, without modification,
  10  *    this list of conditions, and the following disclaimer.
  11  * 2. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  18  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 /*
  28  * Platform (FreeBSD) dependent common attachment code for Qlogic adapters.
  29  */
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 #include <dev/isp/isp_freebsd.h>
  34 #include <sys/unistd.h>
  35 #include <sys/kthread.h>
  36 #include <sys/conf.h>
  37 #include <sys/module.h>
  38 #include <sys/ioccom.h>
  39 #include <dev/isp/isp_ioctl.h>
  40 #include <sys/devicestat.h>
  41 #include <cam/cam_periph.h>
  42 #include <cam/cam_xpt_periph.h>
  43
  44 #if     __FreeBSD_version < 800002
  45 #define THREAD_CREATE   kthread_create
  46 #else
  47 #define THREAD_CREATE   kproc_create
  48 #endif
  49
  50 MODULE_VERSION(isp, 1);
  51 MODULE_DEPEND(isp, cam, 1, 1, 1);
  52 int isp_announced = 0;
  53 int isp_fabric_hysteresis = 5;
  54 int isp_loop_down_limit = 60;   /* default loop down limit */
  55 int isp_change_is_bad = 0;      /* "changed" devices are bad */
  56 int isp_quickboot_time = 7;     /* don't wait more than N secs for loop up */
  57 int isp_gone_device_time = 30;  /* grace time before reporting device lost */
  58 int isp_autoconfig = 1;         /* automatically attach/detach devices */
  59 static const char prom3[] = "Chan %d PortID 0x%06x Departed from Target %u because of %s";
  60
  61 static void isp_freeze_loopdown(ispsoftc_t *, int, char *);
  62 static d_ioctl_t ispioctl;
  63 static void isp_intr_enable(void *);
  64 static void isp_cam_async(void *, uint32_t, struct cam_path *, void *);
  65 static void isp_poll(struct cam_sim *);
  66 static timeout_t isp_watchdog;
  67 static timeout_t isp_gdt;
  68 static task_fn_t isp_gdt_task;
  69 static timeout_t isp_ldt;
  70 static task_fn_t isp_ldt_task;
  71 static void isp_kthread(void *);
  72 static void isp_action(struct cam_sim *, union ccb *);
  73 #ifdef  ISP_INTERNAL_TARGET
  74 static void isp_target_thread_pi(void *);
  75 static void isp_target_thread_fc(void *);
  76 #endif
  77 static int isp_timer_count;
  78 static void isp_timer(void *);
  79
  80 static struct cdevsw isp_cdevsw = {
  81         .d_version =    D_VERSION,
  82         .d_ioctl =      ispioctl,
  83         .d_name =       "isp",
  84 };
  85
  86 static int
  87 isp_role_sysctl(SYSCTL_HANDLER_ARGS)
  88 {
  89         ispsoftc_t *isp = (ispsoftc_t *)arg1;
  90         int chan = arg2;
  91         int error, old, value;
  92
  93         value = FCPARAM(isp, chan)->role;
  94
  95         error = sysctl_handle_int(oidp, &value, 0, req);
  96         if ((error != 0) || (req->newptr == NULL))
  97                 return (error);
  98
  99         if (value < ISP_ROLE_NONE || value > ISP_ROLE_BOTH)
 100                 return (EINVAL);
 101
 102         ISP_LOCK(isp);
 103         old = FCPARAM(isp, chan)->role;
 104
 105         /* If nothing has changed -- we are done. */
 106         if (value == old) {
 107                 ISP_UNLOCK(isp);
 108                 return (0);
 109         }
 110
 111         /* We don't allow target mode switch from here. */
 112         if ((value ^ old) & ISP_ROLE_TARGET) {
 113                 ISP_UNLOCK(isp);
 114                 return (EPERM);
 115         }
 116
 117         /* Actually change the role. */
 118         error = isp_fc_change_role(isp, chan, value);
 119         ISP_UNLOCK(isp);
 120         return (error);
 121 }
 122
 123 static int
 124 isp_attach_chan(ispsoftc_t *isp, struct cam_devq *devq, int chan)
 125 {
 126         struct ccb_setasync csa;
 127         struct cam_sim *sim;
 128         struct cam_path *path;
 129
 130         /*
 131          * Construct our SIM entry.
 132          */
 133         sim = cam_sim_alloc(isp_action, isp_poll, "isp", isp, device_get_unit(isp->isp_dev), &isp->isp_osinfo.lock, isp->isp_maxcmds, isp->isp_maxcmds, devq);
 134
 135         if (sim == NULL) {
 136                 return (ENOMEM);
 137         }
 138
 139         ISP_LOCK(isp);
 140         if (xpt_bus_register(sim, isp->isp_dev, chan) != CAM_SUCCESS) {
 141                 ISP_UNLOCK(isp);
 142                 cam_sim_free(sim, FALSE);
 143                 return (EIO);
 144         }
 145         ISP_UNLOCK(isp);
 146         if (xpt_create_path(&path, NULL, cam_sim_path(sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 147                 ISP_LOCK(isp);
 148                 xpt_bus_deregister(cam_sim_path(sim));
 149                 ISP_UNLOCK(isp);
 150                 cam_sim_free(sim, FALSE);
 151                 return (ENXIO);
 152         }
 153         xpt_setup_ccb(&csa.ccb_h, path, 5);
 154         csa.ccb_h.func_code = XPT_SASYNC_CB;
 155         csa.event_enable = AC_LOST_DEVICE;
 156         csa.callback = isp_cam_async;
 157         csa.callback_arg = sim;
 158
 159         ISP_LOCK(isp);
 160         xpt_action((union ccb *)&csa);
 161         ISP_UNLOCK(isp);
 162
 163         if (IS_SCSI(isp)) {
 164                 struct isp_spi *spi = ISP_SPI_PC(isp, chan);
 165                 spi->sim = sim;
 166                 spi->path = path;
 167 #ifdef  ISP_INTERNAL_TARGET
 168                 ISP_SET_PC(isp, chan, proc_active, 1);
 169                 if (THREAD_CREATE(isp_target_thread_pi, spi, &spi->target_proc, 0, 0, "%s: isp_test_tgt%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) {
 170                         ISP_SET_PC(isp, chan, proc_active, 0);
 171                         isp_prt(isp, ISP_LOGERR, "cannot create test target thread");
 172                 }
 173 #endif
 174         } else {
 175                 fcparam *fcp = FCPARAM(isp, chan);
 176                 struct isp_fc *fc = ISP_FC_PC(isp, chan);
 177                 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(isp->isp_osinfo.dev);
 178                 struct sysctl_oid *tree = device_get_sysctl_tree(isp->isp_osinfo.dev);
 179                 char name[16];
 180
 181                 ISP_LOCK(isp);
 182                 fc->sim = sim;
 183                 fc->path = path;
 184                 fc->isp = isp;
 185                 fc->ready = 1;
 186
 187                 callout_init_mtx(&fc->ldt, &isp->isp_osinfo.lock, 0);
 188                 callout_init_mtx(&fc->gdt, &isp->isp_osinfo.lock, 0);
 189                 TASK_INIT(&fc->ltask, 1, isp_ldt_task, fc);
 190                 TASK_INIT(&fc->gtask, 1, isp_gdt_task, fc);
 191
 192                 /*
 193                  * We start by being "loop down" if we have an initiator role
 194                  */
 195                 if (fcp->role & ISP_ROLE_INITIATOR) {
 196                         isp_freeze_loopdown(isp, chan, "isp_attach");
 197                         callout_reset(&fc->ldt, isp_quickboot_time * hz, isp_ldt, fc);
 198                         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Starting Initial Loop Down Timer @ %lu", (unsigned long) time_uptime);
 199                 }
 200                 ISP_UNLOCK(isp);
 201                 if (THREAD_CREATE(isp_kthread, fc, &fc->kproc, 0, 0, "%s: fc_thrd%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) {
 202                         xpt_free_path(fc->path);
 203                         ISP_LOCK(isp);
 204                         if (callout_active(&fc->ldt))
 205                                 callout_stop(&fc->ldt);
 206                         xpt_bus_deregister(cam_sim_path(fc->sim));
 207                         ISP_UNLOCK(isp);
 208                         cam_sim_free(fc->sim, FALSE);
 209                         return (ENOMEM);
 210                 }
 211 #ifdef  ISP_INTERNAL_TARGET
 212                 ISP_SET_PC(isp, chan, proc_active, 1);
 213                 if (THREAD_CREATE(isp_target_thread_fc, fc, &fc->target_proc, 0, 0, "%s: isp_test_tgt%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) {
 214                         ISP_SET_PC(isp, chan, proc_active, 0);
 215                         isp_prt(isp, ISP_LOGERR, "cannot create test target thread");
 216                 }
 217 #endif
 218                 if (chan > 0) {
 219                         snprintf(name, sizeof(name), "chan%d", chan);
 220                         tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(tree),
 221                             OID_AUTO, name, CTLFLAG_RW, 0, "Virtual channel");
 222                 }
 223                 SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwnn", CTLFLAG_RD, &FCPARAM(isp, chan)->isp_wwnn, "World Wide Node Name");
 224                 SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "wwpn", CTLFLAG_RD, &FCPARAM(isp, chan)->isp_wwpn, "World Wide Port Name");
 225                 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "loop_down_limit", CTLFLAG_RW, &ISP_FC_PC(isp, chan)->loop_down_limit, 0, "Loop Down Limit");
 226                 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "gone_device_time", CTLFLAG_RW, &ISP_FC_PC(isp, chan)->gone_device_time, 0, "Gone Device Time");
 227 #if defined(ISP_TARGET_MODE) && defined(DEBUG)
 228                 SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "inject_lost_data_frame", CTLFLAG_RW, &ISP_FC_PC(isp, chan)->inject_lost_data_frame, 0, "Cause a Lost Frame on a Read");
 229 #endif
 230                 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 231                     "role", CTLTYPE_INT | CTLFLAG_RW, isp, chan,
 232                     isp_role_sysctl, "I", "Current role");
 233         }
 234         return (0);
 235 }
 236
 237 int
 238 isp_attach(ispsoftc_t *isp)
 239 {
 240         const char *nu = device_get_nameunit(isp->isp_osinfo.dev);
 241         int du = device_get_unit(isp->isp_dev);
 242         int chan;
 243
 244         isp->isp_osinfo.ehook.ich_func = isp_intr_enable;
 245         isp->isp_osinfo.ehook.ich_arg = isp;
 246         /*
 247          * Haha. Set this first, because if we're loaded as a module isp_intr_enable
 248          * will be called right awawy, which will clear isp_osinfo.ehook_active,
 249          * which would be unwise to then set again later.
 250          */
 251         isp->isp_osinfo.ehook_active = 1;
 252         if (config_intrhook_establish(&isp->isp_osinfo.ehook) != 0) {
 253                 isp_prt(isp, ISP_LOGERR, "could not establish interrupt enable hook");
 254                 return (-EIO);
 255         }
 256
 257         /*
 258          * Create the device queue for our SIM(s).
 259          */
 260         isp->isp_osinfo.devq = cam_simq_alloc(isp->isp_maxcmds);
 261         if (isp->isp_osinfo.devq == NULL) {
 262                 config_intrhook_disestablish(&isp->isp_osinfo.ehook);
 263                 return (EIO);
 264         }
 265
 266         for (chan = 0; chan < isp->isp_nchan; chan++) {
 267                 if (isp_attach_chan(isp, isp->isp_osinfo.devq, chan)) {
 268                         goto unwind;
 269                 }
 270         }
 271
 272         callout_init_mtx(&isp->isp_osinfo.tmo, &isp->isp_osinfo.lock, 0);
 273         isp_timer_count = hz >> 2;
 274         callout_reset(&isp->isp_osinfo.tmo, isp_timer_count, isp_timer, isp);
 275         isp->isp_osinfo.timer_active = 1;
 276
 277         isp->isp_osinfo.cdev = make_dev(&isp_cdevsw, du, UID_ROOT, GID_OPERATOR, 0600, "%s", nu);
 278         if (isp->isp_osinfo.cdev) {
 279                 isp->isp_osinfo.cdev->si_drv1 = isp;
 280         }
 281         return (0);
 282
 283 unwind:
 284         while (--chan >= 0) {
 285                 struct cam_sim *sim;
 286                 struct cam_path *path;
 287                 if (IS_FC(isp)) {
 288                         sim = ISP_FC_PC(isp, chan)->sim;
 289                         path = ISP_FC_PC(isp, chan)->path;
 290                 } else {
 291                         sim = ISP_SPI_PC(isp, chan)->sim;
 292                         path = ISP_SPI_PC(isp, chan)->path;
 293                 }
 294                 xpt_free_path(path);
 295                 ISP_LOCK(isp);
 296                 xpt_bus_deregister(cam_sim_path(sim));
 297                 ISP_UNLOCK(isp);
 298                 cam_sim_free(sim, FALSE);
 299         }
 300         if (isp->isp_osinfo.ehook_active) {
 301                 config_intrhook_disestablish(&isp->isp_osinfo.ehook);
 302                 isp->isp_osinfo.ehook_active = 0;
 303         }
 304         if (isp->isp_osinfo.cdev) {
 305                 destroy_dev(isp->isp_osinfo.cdev);
 306                 isp->isp_osinfo.cdev = NULL;
 307         }
 308         cam_simq_free(isp->isp_osinfo.devq);
 309         isp->isp_osinfo.devq = NULL;
 310         return (-1);
 311 }
 312
 313 int
 314 isp_detach(ispsoftc_t *isp)
 315 {
 316         struct cam_sim *sim;
 317         struct cam_path *path;
 318         struct ccb_setasync csa;
 319         int chan;
 320
 321         ISP_LOCK(isp);
 322         for (chan = isp->isp_nchan - 1; chan >= 0; chan -= 1) {
 323                 if (IS_FC(isp)) {
 324                         sim = ISP_FC_PC(isp, chan)->sim;
 325                         path = ISP_FC_PC(isp, chan)->path;
 326                 } else {
 327                         sim = ISP_SPI_PC(isp, chan)->sim;
 328                         path = ISP_SPI_PC(isp, chan)->path;
 329                 }
 330                 if (sim->refcount > 2) {
 331                         ISP_UNLOCK(isp);
 332                         return (EBUSY);
 333                 }
 334         }
 335         if (isp->isp_osinfo.timer_active) {
 336                 callout_stop(&isp->isp_osinfo.tmo);
 337                 isp->isp_osinfo.timer_active = 0;
 338         }
 339         for (chan = isp->isp_nchan - 1; chan >= 0; chan -= 1) {
 340                 if (IS_FC(isp)) {
 341                         sim = ISP_FC_PC(isp, chan)->sim;
 342                         path = ISP_FC_PC(isp, chan)->path;
 343                 } else {
 344                         sim = ISP_SPI_PC(isp, chan)->sim;
 345                         path = ISP_SPI_PC(isp, chan)->path;
 346                 }
 347                 xpt_setup_ccb(&csa.ccb_h, path, 5);
 348                 csa.ccb_h.func_code = XPT_SASYNC_CB;
 349                 csa.event_enable = 0;
 350                 csa.callback = isp_cam_async;
 351                 csa.callback_arg = sim;
 352                 ISP_LOCK(isp);
 353                 xpt_action((union ccb *)&csa);
 354                 ISP_UNLOCK(isp);
 355                 xpt_free_path(path);
 356                 xpt_bus_deregister(cam_sim_path(sim));
 357                 cam_sim_free(sim, FALSE);
 358         }
 359         ISP_UNLOCK(isp);
 360         if (isp->isp_osinfo.cdev) {
 361                 destroy_dev(isp->isp_osinfo.cdev);
 362                 isp->isp_osinfo.cdev = NULL;
 363         }
 364         if (isp->isp_osinfo.ehook_active) {
 365                 config_intrhook_disestablish(&isp->isp_osinfo.ehook);
 366                 isp->isp_osinfo.ehook_active = 0;
 367         }
 368         if (isp->isp_osinfo.devq != NULL) {
 369                 cam_simq_free(isp->isp_osinfo.devq);
 370                 isp->isp_osinfo.devq = NULL;
 371         }
 372         return (0);
 373 }
 374
 375 static void
 376 isp_freeze_loopdown(ispsoftc_t *isp, int chan, char *msg)
 377 {
 378         if (IS_FC(isp)) {
 379                 struct isp_fc *fc = ISP_FC_PC(isp, chan);
 380                 if (fc->simqfrozen == 0) {
 381                         isp_prt(isp, ISP_LOGDEBUG0, "%s: freeze simq (loopdown) chan %d", msg, chan);
 382                         fc->simqfrozen = SIMQFRZ_LOOPDOWN;
 383                         xpt_freeze_simq(fc->sim, 1);
 384                 } else {
 385                         isp_prt(isp, ISP_LOGDEBUG0, "%s: mark frozen (loopdown) chan %d", msg, chan);
 386                         fc->simqfrozen |= SIMQFRZ_LOOPDOWN;
 387                 }
 388         }
 389 }
 390
 391 static void
 392 isp_unfreeze_loopdown(ispsoftc_t *isp, int chan)
 393 {
 394         if (IS_FC(isp)) {
 395                 struct isp_fc *fc = ISP_FC_PC(isp, chan);
 396                 int wasfrozen = fc->simqfrozen & SIMQFRZ_LOOPDOWN;
 397                 fc->simqfrozen &= ~SIMQFRZ_LOOPDOWN;
 398                 if (wasfrozen && fc->simqfrozen == 0) {
 399                         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d releasing simq", __func__, chan);
 400                         xpt_release_simq(fc->sim, 1);
 401                 }
 402         }
 403 }
 404
 405
 406 static int
 407 ispioctl(struct cdev *dev, u_long c, caddr_t addr, int flags, struct thread *td)
 408 {
 409         ispsoftc_t *isp;
 410         int nr, chan, retval = ENOTTY;
 411
 412         isp = dev->si_drv1;
 413
 414         switch (c) {
 415         case ISP_SDBLEV:
 416         {
 417                 int olddblev = isp->isp_dblev;
 418                 isp->isp_dblev = *(int *)addr;
 419                 *(int *)addr = olddblev;
 420                 retval = 0;
 421                 break;
 422         }
 423         case ISP_GETROLE:
 424                 chan = *(int *)addr;
 425                 if (chan < 0 || chan >= isp->isp_nchan) {
 426                         retval = -ENXIO;
 427                         break;
 428                 }
 429                 if (IS_FC(isp)) {
 430                         *(int *)addr = FCPARAM(isp, chan)->role;
 431                 } else {
 432                         *(int *)addr = SDPARAM(isp, chan)->role;
 433                 }
 434                 retval = 0;
 435                 break;
 436         case ISP_SETROLE:
 437                 nr = *(int *)addr;
 438                 chan = nr >> 8;
 439                 if (chan < 0 || chan >= isp->isp_nchan) {
 440                         retval = -ENXIO;
 441                         break;
 442                 }
 443                 nr &= 0xff;
 444                 if (nr & ~(ISP_ROLE_INITIATOR|ISP_ROLE_TARGET)) {
 445                         retval = EINVAL;
 446                         break;
 447                 }
 448                 if (IS_FC(isp)) {
 449                         /*
 450                          * We don't really support dual role at present on FC cards.
 451                          *
 452                          * We should, but a bunch of things are currently broken,
 453                          * so don't allow it.
 454                          */
 455                         if (nr == ISP_ROLE_BOTH) {
 456                                 isp_prt(isp, ISP_LOGERR, "cannot support dual role at present");
 457                                 retval = EINVAL;
 458                                 break;
 459                         }
 460                         *(int *)addr = FCPARAM(isp, chan)->role;
 461 #ifdef  ISP_INTERNAL_TARGET
 462                         ISP_LOCK(isp);
 463                         retval = isp_fc_change_role(isp, chan, nr);
 464                         ISP_UNLOCK(isp);
 465 #else
 466                         FCPARAM(isp, chan)->role = nr;
 467 #endif
 468                 } else {
 469                         *(int *)addr = SDPARAM(isp, chan)->role;
 470                         SDPARAM(isp, chan)->role = nr;
 471                 }
 472                 retval = 0;
 473                 break;
 474
 475         case ISP_RESETHBA:
 476                 ISP_LOCK(isp);
 477 #ifdef  ISP_TARGET_MODE
 478                 isp_del_all_wwn_entries(isp, ISP_NOCHAN);
 479 #endif
 480                 isp_reinit(isp, 0);
 481                 ISP_UNLOCK(isp);
 482                 retval = 0;
 483                 break;
 484
 485         case ISP_RESCAN:
 486                 if (IS_FC(isp)) {
 487                         chan = *(int *)addr;
 488                         if (chan < 0 || chan >= isp->isp_nchan) {
 489                                 retval = -ENXIO;
 490                                 break;
 491                         }
 492                         ISP_LOCK(isp);
 493                         if (isp_fc_runstate(isp, chan, 5 * 1000000)) {
 494                                 retval = EIO;
 495                         } else {
 496                                 retval = 0;
 497                         }
 498                         ISP_UNLOCK(isp);
 499                 }
 500                 break;
 501
 502         case ISP_FC_LIP:
 503                 if (IS_FC(isp)) {
 504                         chan = *(int *)addr;
 505                         if (chan < 0 || chan >= isp->isp_nchan) {
 506                                 retval = -ENXIO;
 507                                 break;
 508                         }
 509                         ISP_LOCK(isp);
 510                         if (isp_control(isp, ISPCTL_SEND_LIP, chan)) {
 511                                 retval = EIO;
 512                         } else {
 513                                 retval = 0;
 514                         }
 515                         ISP_UNLOCK(isp);
 516                 }
 517                 break;
 518         case ISP_FC_GETDINFO:
 519         {
 520                 struct isp_fc_device *ifc = (struct isp_fc_device *) addr;
 521                 fcportdb_t *lp;
 522
 523                 if (IS_SCSI(isp)) {
 524                         break;
 525                 }
 526                 if (ifc->loopid >= MAX_FC_TARG) {
 527                         retval = EINVAL;
 528                         break;
 529                 }
 530                 lp = &FCPARAM(isp, ifc->chan)->portdb[ifc->loopid];
 531                 if (lp->state == FC_PORTDB_STATE_VALID || lp->target_mode) {
 532                         ifc->role = (lp->prli_word3 & SVC3_ROLE_MASK) >> SVC3_ROLE_SHIFT;
 533                         ifc->loopid = lp->handle;
 534                         ifc->portid = lp->portid;
 535                         ifc->node_wwn = lp->node_wwn;
 536                         ifc->port_wwn = lp->port_wwn;
 537                         retval = 0;
 538                 } else {
 539                         retval = ENODEV;
 540                 }
 541                 break;
 542         }
 543         case ISP_GET_STATS:
 544         {
 545                 isp_stats_t *sp = (isp_stats_t *) addr;
 546
 547                 ISP_MEMZERO(sp, sizeof (*sp));
 548                 sp->isp_stat_version = ISP_STATS_VERSION;
 549                 sp->isp_type = isp->isp_type;
 550                 sp->isp_revision = isp->isp_revision;
 551                 ISP_LOCK(isp);
 552                 sp->isp_stats[ISP_INTCNT] = isp->isp_intcnt;
 553                 sp->isp_stats[ISP_INTBOGUS] = isp->isp_intbogus;
 554                 sp->isp_stats[ISP_INTMBOXC] = isp->isp_intmboxc;
 555                 sp->isp_stats[ISP_INGOASYNC] = isp->isp_intoasync;
 556                 sp->isp_stats[ISP_RSLTCCMPLT] = isp->isp_rsltccmplt;
 557                 sp->isp_stats[ISP_FPHCCMCPLT] = isp->isp_fphccmplt;
 558                 sp->isp_stats[ISP_RSCCHIWAT] = isp->isp_rscchiwater;
 559                 sp->isp_stats[ISP_FPCCHIWAT] = isp->isp_fpcchiwater;
 560                 ISP_UNLOCK(isp);
 561                 retval = 0;
 562                 break;
 563         }
 564         case ISP_CLR_STATS:
 565                 ISP_LOCK(isp);
 566                 isp->isp_intcnt = 0;
 567                 isp->isp_intbogus = 0;
 568                 isp->isp_intmboxc = 0;
 569                 isp->isp_intoasync = 0;
 570                 isp->isp_rsltccmplt = 0;
 571                 isp->isp_fphccmplt = 0;
 572                 isp->isp_rscchiwater = 0;
 573                 isp->isp_fpcchiwater = 0;
 574                 ISP_UNLOCK(isp);
 575                 retval = 0;
 576                 break;
 577         case ISP_FC_GETHINFO:
 578         {
 579                 struct isp_hba_device *hba = (struct isp_hba_device *) addr;
 580                 int chan = hba->fc_channel;
 581
 582                 if (chan < 0 || chan >= isp->isp_nchan) {
 583                         retval = ENXIO;
 584                         break;
 585                 }
 586                 hba->fc_fw_major = ISP_FW_MAJORX(isp->isp_fwrev);
 587                 hba->fc_fw_minor = ISP_FW_MINORX(isp->isp_fwrev);
 588                 hba->fc_fw_micro = ISP_FW_MICROX(isp->isp_fwrev);
 589                 hba->fc_nchannels = isp->isp_nchan;
 590                 if (IS_FC(isp)) {
 591                         hba->fc_nports = MAX_FC_TARG;
 592                         hba->fc_speed = FCPARAM(isp, hba->fc_channel)->isp_gbspeed;
 593                         hba->fc_topology = FCPARAM(isp, chan)->isp_topo + 1;
 594                         hba->fc_loopid = FCPARAM(isp, chan)->isp_loopid;
 595                         hba->nvram_node_wwn = FCPARAM(isp, chan)->isp_wwnn_nvram;
 596                         hba->nvram_port_wwn = FCPARAM(isp, chan)->isp_wwpn_nvram;
 597                         hba->active_node_wwn = FCPARAM(isp, chan)->isp_wwnn;
 598                         hba->active_port_wwn = FCPARAM(isp, chan)->isp_wwpn;
 599                 } else {
 600                         hba->fc_nports = MAX_TARGETS;
 601                         hba->fc_speed = 0;
 602                         hba->fc_topology = 0;
 603                         hba->nvram_node_wwn = 0ull;
 604                         hba->nvram_port_wwn = 0ull;
 605                         hba->active_node_wwn = 0ull;
 606                         hba->active_port_wwn = 0ull;
 607                 }
 608                 retval = 0;
 609                 break;
 610         }
 611         case ISP_TSK_MGMT:
 612         {
 613                 int needmarker;
 614                 struct isp_fc_tsk_mgmt *fct = (struct isp_fc_tsk_mgmt *) addr;
 615                 uint16_t loopid;
 616                 mbreg_t mbs;
 617
 618                 if (IS_SCSI(isp)) {
 619                         break;
 620                 }
 621
 622                 chan = fct->chan;
 623                 if (chan < 0 || chan >= isp->isp_nchan) {
 624                         retval = -ENXIO;
 625                         break;
 626                 }
 627
 628                 needmarker = retval = 0;
 629                 loopid = fct->loopid;
 630                 ISP_LOCK(isp);
 631                 if (IS_24XX(isp)) {
 632                         uint8_t local[QENTRY_LEN];
 633                         isp24xx_tmf_t *tmf;
 634                         isp24xx_statusreq_t *sp;
 635                         fcparam *fcp = FCPARAM(isp, chan);
 636                         fcportdb_t *lp;
 637                         int i;
 638
 639                         for (i = 0; i < MAX_FC_TARG; i++) {
 640                                 lp = &fcp->portdb[i];
 641                                 if (lp->handle == loopid) {
 642                                         break;
 643                                 }
 644                         }
 645                         if (i == MAX_FC_TARG) {
 646                                 retval = ENXIO;
 647                                 ISP_UNLOCK(isp);
 648                                 break;
 649                         }
 650                         /* XXX VALIDATE LP XXX */
 651                         tmf = (isp24xx_tmf_t *) local;
 652                         ISP_MEMZERO(tmf, QENTRY_LEN);
 653                         tmf->tmf_header.rqs_entry_type = RQSTYPE_TSK_MGMT;
 654                         tmf->tmf_header.rqs_entry_count = 1;
 655                         tmf->tmf_nphdl = lp->handle;
 656                         tmf->tmf_delay = 2;
 657                         tmf->tmf_timeout = 2;
 658                         tmf->tmf_tidlo = lp->portid;
 659                         tmf->tmf_tidhi = lp->portid >> 16;
 660                         tmf->tmf_vpidx = ISP_GET_VPIDX(isp, chan);
 661                         tmf->tmf_lun[1] = fct->lun & 0xff;
 662                         if (fct->lun >= 256) {
 663                                 tmf->tmf_lun[0] = 0x40 | (fct->lun >> 8);
 664                         }
 665                         switch (fct->action) {
 666                         case IPT_CLEAR_ACA:
 667                                 tmf->tmf_flags = ISP24XX_TMF_CLEAR_ACA;
 668                                 break;
 669                         case IPT_TARGET_RESET:
 670                                 tmf->tmf_flags = ISP24XX_TMF_TARGET_RESET;
 671                                 needmarker = 1;
 672                                 break;
 673                         case IPT_LUN_RESET:
 674                                 tmf->tmf_flags = ISP24XX_TMF_LUN_RESET;
 675                                 needmarker = 1;
 676                                 break;
 677                         case IPT_CLEAR_TASK_SET:
 678                                 tmf->tmf_flags = ISP24XX_TMF_CLEAR_TASK_SET;
 679                                 needmarker = 1;
 680                                 break;
 681                         case IPT_ABORT_TASK_SET:
 682                                 tmf->tmf_flags = ISP24XX_TMF_ABORT_TASK_SET;
 683                                 needmarker = 1;
 684                                 break;
 685                         default:
 686                                 retval = EINVAL;
 687                                 break;
 688                         }
 689                         if (retval) {
 690                                 ISP_UNLOCK(isp);
 691                                 break;
 692                         }
 693                         MBSINIT(&mbs, MBOX_EXEC_COMMAND_IOCB_A64, MBLOGALL, 5000000);
 694                         mbs.param[1] = QENTRY_LEN;
 695                         mbs.param[2] = DMA_WD1(fcp->isp_scdma);
 696                         mbs.param[3] = DMA_WD0(fcp->isp_scdma);
 697                         mbs.param[6] = DMA_WD3(fcp->isp_scdma);
 698                         mbs.param[7] = DMA_WD2(fcp->isp_scdma);
 699
 700                         if (FC_SCRATCH_ACQUIRE(isp, chan)) {
 701                                 ISP_UNLOCK(isp);
 702                                 retval = ENOMEM;
 703                                 break;
 704                         }
 705                         isp_put_24xx_tmf(isp, tmf, fcp->isp_scratch);
 706                         MEMORYBARRIER(isp, SYNC_SFORDEV, 0, QENTRY_LEN, chan);
 707                         sp = (isp24xx_statusreq_t *) local;
 708                         sp->req_completion_status = 1;
 709                         retval = isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs);
 710                         MEMORYBARRIER(isp, SYNC_SFORCPU, QENTRY_LEN, QENTRY_LEN, chan);
 711                         isp_get_24xx_response(isp, &((isp24xx_statusreq_t *)fcp->isp_scratch)[1], sp);
 712                         FC_SCRATCH_RELEASE(isp, chan);
 713                         if (retval || sp->req_completion_status != 0) {
 714                                 FC_SCRATCH_RELEASE(isp, chan);
 715                                 retval = EIO;
 716                         }
 717                         if (retval == 0) {
 718                                 if (needmarker) {
 719                                         fcp->sendmarker = 1;
 720                                 }
 721                         }
 722                 } else {
 723                         MBSINIT(&mbs, 0, MBLOGALL, 0);
 724                         if (ISP_CAP_2KLOGIN(isp) == 0) {
 725                                 loopid <<= 8;
 726                         }
 727                         switch (fct->action) {
 728                         case IPT_CLEAR_ACA:
 729                                 mbs.param[0] = MBOX_CLEAR_ACA;
 730                                 mbs.param[1] = loopid;
 731                                 mbs.param[2] = fct->lun;
 732                                 break;
 733                         case IPT_TARGET_RESET:
 734                                 mbs.param[0] = MBOX_TARGET_RESET;
 735                                 mbs.param[1] = loopid;
 736                                 needmarker = 1;
 737                                 break;
 738                         case IPT_LUN_RESET:
 739                                 mbs.param[0] = MBOX_LUN_RESET;
 740                                 mbs.param[1] = loopid;
 741                                 mbs.param[2] = fct->lun;
 742                                 needmarker = 1;
 743                                 break;
 744                         case IPT_CLEAR_TASK_SET:
 745                                 mbs.param[0] = MBOX_CLEAR_TASK_SET;
 746                                 mbs.param[1] = loopid;
 747                                 mbs.param[2] = fct->lun;
 748                                 needmarker = 1;
 749                                 break;
 750                         case IPT_ABORT_TASK_SET:
 751                                 mbs.param[0] = MBOX_ABORT_TASK_SET;
 752                                 mbs.param[1] = loopid;
 753                                 mbs.param[2] = fct->lun;
 754                                 needmarker = 1;
 755                                 break;
 756                         default:
 757                                 retval = EINVAL;
 758                                 break;
 759                         }
 760                         if (retval == 0) {
 761                                 if (needmarker) {
 762                                         FCPARAM(isp, chan)->sendmarker = 1;
 763                                 }
 764                                 retval = isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs);
 765                                 if (retval) {
 766                                         retval = EIO;
 767                                 }
 768                         }
 769                 }
 770                 ISP_UNLOCK(isp);
 771                 break;
 772         }
 773         default:
 774                 break;
 775         }
 776         return (retval);
 777 }
 778
 779 static void
 780 isp_intr_enable(void *arg)
 781 {
 782         int chan;
 783         ispsoftc_t *isp = arg;
 784         ISP_LOCK(isp);
 785         for (chan = 0; chan < isp->isp_nchan; chan++) {
 786                 if (IS_FC(isp)) {
 787                         if (FCPARAM(isp, chan)->role != ISP_ROLE_NONE) {
 788                                 ISP_ENABLE_INTS(isp);
 789                                 break;
 790                         }
 791                 } else {
 792                         if (SDPARAM(isp, chan)->role != ISP_ROLE_NONE) {
 793                                 ISP_ENABLE_INTS(isp);
 794                                 break;
 795                         }
 796                 }
 797         }
 798         isp->isp_osinfo.ehook_active = 0;
 799         ISP_UNLOCK(isp);
 800         /* Release our hook so that the boot can continue. */
 801         config_intrhook_disestablish(&isp->isp_osinfo.ehook);
 802 }
 803
 804 /*
 805  * Local Inlines
 806  */
 807
 808 static ISP_INLINE int isp_get_pcmd(ispsoftc_t *, union ccb *);
 809 static ISP_INLINE void isp_free_pcmd(ispsoftc_t *, union ccb *);
 810
 811 static ISP_INLINE int
 812 isp_get_pcmd(ispsoftc_t *isp, union ccb *ccb)
 813 {
 814         ISP_PCMD(ccb) = isp->isp_osinfo.pcmd_free;
 815         if (ISP_PCMD(ccb) == NULL) {
 816                 return (-1);
 817         }
 818         isp->isp_osinfo.pcmd_free = ((struct isp_pcmd *)ISP_PCMD(ccb))->next;
 819         return (0);
 820 }
 821
 822 static ISP_INLINE void
 823 isp_free_pcmd(ispsoftc_t *isp, union ccb *ccb)
 824 {
 825         if (ISP_PCMD(ccb)) {
 826 #ifdef  ISP_TARGET_MODE
 827                 PISP_PCMD(ccb)->datalen = 0;
 828                 PISP_PCMD(ccb)->totslen = 0;
 829                 PISP_PCMD(ccb)->cumslen = 0;
 830                 PISP_PCMD(ccb)->crn = 0;
 831 #endif
 832                 PISP_PCMD(ccb)->next = isp->isp_osinfo.pcmd_free;
 833                 isp->isp_osinfo.pcmd_free = ISP_PCMD(ccb);
 834                 ISP_PCMD(ccb) = NULL;
 835         }
 836 }
 837
 838 /*
 839  * Put the target mode functions here, because some are inlines
 840  */
 841 #ifdef  ISP_TARGET_MODE
 842 static ISP_INLINE void isp_tmlock(ispsoftc_t *, const char *);
 843 static ISP_INLINE void isp_tmunlk(ispsoftc_t *);
 844 static ISP_INLINE int is_any_lun_enabled(ispsoftc_t *, int);
 845 static ISP_INLINE int is_lun_enabled(ispsoftc_t *, int, lun_id_t);
 846 static ISP_INLINE tstate_t *get_lun_statep(ispsoftc_t *, int, lun_id_t);
 847 static ISP_INLINE tstate_t *get_lun_statep_from_tag(ispsoftc_t *, int, uint32_t);
 848 static ISP_INLINE void rls_lun_statep(ispsoftc_t *, tstate_t *);
 849 static ISP_INLINE inot_private_data_t *get_ntp_from_tagdata(ispsoftc_t *, uint32_t, uint32_t, tstate_t **);
 850 static ISP_INLINE atio_private_data_t *isp_get_atpd(ispsoftc_t *, tstate_t *, uint32_t);
 851 static ISP_INLINE atio_private_data_t *isp_find_atpd(ispsoftc_t *, tstate_t *, uint32_t);
 852 static ISP_INLINE void isp_put_atpd(ispsoftc_t *, tstate_t *, atio_private_data_t *);
 853 static ISP_INLINE inot_private_data_t *isp_get_ntpd(ispsoftc_t *, tstate_t *);
 854 static ISP_INLINE inot_private_data_t *isp_find_ntpd(ispsoftc_t *, tstate_t *, uint32_t, uint32_t);
 855 static ISP_INLINE void isp_put_ntpd(ispsoftc_t *, tstate_t *, inot_private_data_t *);
 856 static cam_status create_lun_state(ispsoftc_t *, int, struct cam_path *, tstate_t **);
 857 static void destroy_lun_state(ispsoftc_t *, tstate_t *);
 858 static void isp_enable_lun(ispsoftc_t *, union ccb *);
 859 static cam_status isp_enable_deferred_luns(ispsoftc_t *, int);
 860 static cam_status isp_enable_deferred(ispsoftc_t *, int, lun_id_t);
 861 static void isp_disable_lun(ispsoftc_t *, union ccb *);
 862 static int isp_enable_target_mode(ispsoftc_t *, int);
 863 static int isp_disable_target_mode(ispsoftc_t *, int);
 864 static void isp_ledone(ispsoftc_t *, lun_entry_t *);
 865 static timeout_t isp_refire_putback_atio;
 866 static timeout_t isp_refire_notify_ack;
 867 static void isp_complete_ctio(union ccb *);
 868 static void isp_target_putback_atio(union ccb *);
 869 enum Start_Ctio_How { FROM_CAM, FROM_TIMER, FROM_SRR, FROM_CTIO_DONE };
 870 static void isp_target_start_ctio(ispsoftc_t *, union ccb *, enum Start_Ctio_How);
 871 static void isp_handle_platform_atio(ispsoftc_t *, at_entry_t *);
 872 static void isp_handle_platform_atio2(ispsoftc_t *, at2_entry_t *);
 873 static void isp_handle_platform_atio7(ispsoftc_t *, at7_entry_t *);
 874 static void isp_handle_platform_ctio(ispsoftc_t *, void *);
 875 static void isp_handle_platform_notify_scsi(ispsoftc_t *, in_entry_t *);
 876 static void isp_handle_platform_notify_fc(ispsoftc_t *, in_fcentry_t *);
 877 static void isp_handle_platform_notify_24xx(ispsoftc_t *, in_fcentry_24xx_t *);
 878 static int isp_handle_platform_target_notify_ack(ispsoftc_t *, isp_notify_t *);
 879 static void isp_handle_platform_target_tmf(ispsoftc_t *, isp_notify_t *);
 880 static void isp_target_mark_aborted(ispsoftc_t *, union ccb *);
 881 static void isp_target_mark_aborted_early(ispsoftc_t *, tstate_t *, uint32_t);
 882
 883 static ISP_INLINE void
 884 isp_tmlock(ispsoftc_t *isp, const char *msg)
 885 {
 886         while (isp->isp_osinfo.tmbusy) {
 887                 isp->isp_osinfo.tmwanted = 1;
 888                 mtx_sleep(isp, &isp->isp_lock, PRIBIO, msg, 0);
 889         }
 890         isp->isp_osinfo.tmbusy = 1;
 891 }
 892
 893 static ISP_INLINE void
 894 isp_tmunlk(ispsoftc_t *isp)
 895 {
 896         isp->isp_osinfo.tmbusy = 0;
 897         if (isp->isp_osinfo.tmwanted) {
 898                 isp->isp_osinfo.tmwanted = 0;
 899                 wakeup(isp);
 900         }
 901 }
 902
 903 static ISP_INLINE int
 904 is_any_lun_enabled(ispsoftc_t *isp, int bus)
 905 {
 906         struct tslist *lhp;
 907         int i;
 908
 909         for (i = 0; i < LUN_HASH_SIZE; i++) {
 910                 ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
 911                 if (SLIST_FIRST(lhp))
 912                         return (1);
 913         }
 914         return (0);
 915 }
 916
 917 static ISP_INLINE int
 918 is_lun_enabled(ispsoftc_t *isp, int bus, lun_id_t lun)
 919 {
 920         tstate_t *tptr;
 921         struct tslist *lhp;
 922
 923         ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp);
 924         SLIST_FOREACH(tptr, lhp, next) {
 925                 if (tptr->ts_lun == lun) {
 926                         return (1);
 927                 }
 928         }
 929         return (0);
 930 }
 931
 932 static void
 933 dump_tstates(ispsoftc_t *isp, int bus)
 934 {
 935         int i, j;
 936         struct tslist *lhp;
 937         tstate_t *tptr = NULL;
 938
 939         if (bus >= isp->isp_nchan) {
 940                 return;
 941         }
 942         for (i = 0; i < LUN_HASH_SIZE; i++) {
 943                 ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
 944                 j = 0;
 945                 SLIST_FOREACH(tptr, lhp, next) {
 946                         xpt_print(tptr->owner, "[%d, %d] atio_cnt=%d inot_cnt=%d\n", i, j, tptr->atio_count, tptr->inot_count);
 947                         j++;
 948                 }
 949         }
 950 }
 951
 952 static ISP_INLINE tstate_t *
 953 get_lun_statep(ispsoftc_t *isp, int bus, lun_id_t lun)
 954 {
 955         tstate_t *tptr = NULL;
 956         struct tslist *lhp;
 957
 958         if (bus < isp->isp_nchan) {
 959                 ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp);
 960                 SLIST_FOREACH(tptr, lhp, next) {
 961                         if (tptr->ts_lun == lun) {
 962                                 tptr->hold++;
 963                                 return (tptr);
 964                         }
 965                 }
 966         }
 967         return (NULL);
 968 }
 969
 970 static ISP_INLINE tstate_t *
 971 get_lun_statep_from_tag(ispsoftc_t *isp, int bus, uint32_t tagval)
 972 {
 973         tstate_t *tptr = NULL;
 974         atio_private_data_t *atp;
 975         struct tslist *lhp;
 976         int i;
 977
 978         if (bus < isp->isp_nchan && tagval != 0) {
 979                 for (i = 0; i < LUN_HASH_SIZE; i++) {
 980                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
 981                         SLIST_FOREACH(tptr, lhp, next) {
 982                                 atp = isp_find_atpd(isp, tptr, tagval);
 983                                 if (atp) {
 984                                         tptr->hold++;
 985                                         return (tptr);
 986                                 }
 987                         }
 988                 }
 989         }
 990         return (NULL);
 991 }
 992
 993 static ISP_INLINE inot_private_data_t *
 994 get_ntp_from_tagdata(ispsoftc_t *isp, uint32_t tag_id, uint32_t seq_id, tstate_t **rslt)
 995 {
 996         inot_private_data_t *ntp;
 997         tstate_t *tptr;
 998         struct tslist *lhp;
 999         int bus, i;
1000
1001         for (bus = 0; bus < isp->isp_nchan; bus++) {
1002                 for (i = 0; i < LUN_HASH_SIZE; i++) {
1003                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
1004                         SLIST_FOREACH(tptr, lhp, next) {
1005                                 ntp = isp_find_ntpd(isp, tptr, tag_id, seq_id);
1006                                 if (ntp) {
1007                                         *rslt = tptr;
1008                                         tptr->hold++;
1009                                         return (ntp);
1010                                 }
1011                         }
1012                 }
1013         }
1014         return (NULL);
1015 }
1016
1017 static ISP_INLINE void
1018 rls_lun_statep(ispsoftc_t *isp, tstate_t *tptr)
1019 {
1020         KASSERT((tptr->hold), ("tptr not held"));
1021         tptr->hold--;
1022 }
1023
1024 static void
1025 isp_tmcmd_restart(ispsoftc_t *isp)
1026 {
1027         inot_private_data_t *ntp;
1028         inot_private_data_t *restart_queue;
1029         tstate_t *tptr;
1030         union ccb *ccb;
1031         struct tslist *lhp;
1032         int bus, i;
1033
1034         for (bus = 0; bus < isp->isp_nchan; bus++) {
1035                 for (i = 0; i < LUN_HASH_SIZE; i++) {
1036                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
1037                         SLIST_FOREACH(tptr, lhp, next) {
1038                                 if ((restart_queue = tptr->restart_queue) != NULL)
1039                                         tptr->restart_queue = NULL;
1040                                 while (restart_queue) {
1041                                         ntp = restart_queue;
1042                                         restart_queue = ntp->rd.nt.nt_hba;
1043                                         if (IS_24XX(isp)) {
1044                                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at7_entry_t *)ntp->rd.data)->at_rxid);
1045                                                 isp_handle_platform_atio7(isp, (at7_entry_t *) ntp->rd.data);
1046                                         } else {
1047                                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at2_entry_t *)ntp->rd.data)->at_rxid);
1048                                                 isp_handle_platform_atio2(isp, (at2_entry_t *) ntp->rd.data);
1049                                         }
1050                                         isp_put_ntpd(isp, tptr, ntp);
1051                                         if (tptr->restart_queue && restart_queue != NULL) {
1052                                                 ntp = tptr->restart_queue;
1053                                                 tptr->restart_queue = restart_queue;
1054                                                 while (restart_queue->rd.nt.nt_hba) {
1055                                                         restart_queue = restart_queue->rd.nt.nt_hba;
1056                                                 }
1057                                                 restart_queue->rd.nt.nt_hba = ntp;
1058                                                 break;
1059                                         }
1060                                 }
1061                                 /*
1062                                  * We only need to do this once per tptr
1063                                  */
1064                                 if (!TAILQ_EMPTY(&tptr->waitq)) {
1065                                         ccb = (union ccb *)TAILQ_LAST(&tptr->waitq, isp_ccbq);
1066                                         TAILQ_REMOVE(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
1067                                         isp_target_start_ctio(isp, ccb, FROM_TIMER);
1068                                 }
1069                         }
1070                 }
1071         }
1072 }
1073
1074 static ISP_INLINE atio_private_data_t *
1075 isp_get_atpd(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag)
1076 {
1077         atio_private_data_t *atp;
1078
1079         atp = LIST_FIRST(&tptr->atfree);
1080         if (atp) {
1081                 LIST_REMOVE(atp, next);
1082                 atp->tag = tag;
1083                 LIST_INSERT_HEAD(&tptr->atused[ATPDPHASH(tag)], atp, next);
1084         }
1085         return (atp);
1086 }
1087
1088 static ISP_INLINE atio_private_data_t *
1089 isp_find_atpd(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag)
1090 {
1091         atio_private_data_t *atp;
1092
1093         LIST_FOREACH(atp, &tptr->atused[ATPDPHASH(tag)], next) {
1094                 if (atp->tag == tag)
1095                         return (atp);
1096         }
1097         return (NULL);
1098 }
1099
1100 static ISP_INLINE void
1101 isp_put_atpd(ispsoftc_t *isp, tstate_t *tptr, atio_private_data_t *atp)
1102 {
1103         if (atp->ests) {
1104                 isp_put_ecmd(isp, atp->ests);
1105         }
1106         LIST_REMOVE(atp, next);
1107         memset(atp, 0, sizeof (*atp));
1108         LIST_INSERT_HEAD(&tptr->atfree, atp, next);
1109 }
1110
1111 static void
1112 isp_dump_atpd(ispsoftc_t *isp, tstate_t *tptr)
1113 {
1114         atio_private_data_t *atp;
1115         const char *states[8] = { "Free", "ATIO", "CAM", "CTIO", "LAST_CTIO", "PDON", "?6", "7" };
1116
1117         for (atp = tptr->atpool; atp < &tptr->atpool[ATPDPSIZE]; atp++) {
1118                 xpt_print(tptr->owner, "ATP: [0x%x] origdlen %u bytes_xfrd %u lun %u nphdl 0x%04x s_id 0x%06x d_id 0x%06x oxid 0x%04x state %s\n",
1119                     atp->tag, atp->orig_datalen, atp->bytes_xfered, atp->lun, atp->nphdl, atp->sid, atp->portid, atp->oxid, states[atp->state & 0x7]);
1120         }
1121 }
1122
1123
1124 static ISP_INLINE inot_private_data_t *
1125 isp_get_ntpd(ispsoftc_t *isp, tstate_t *tptr)
1126 {
1127         inot_private_data_t *ntp;
1128         ntp = tptr->ntfree;
1129         if (ntp) {
1130                 tptr->ntfree = ntp->next;
1131         }
1132         return (ntp);
1133 }
1134
1135 static ISP_INLINE inot_private_data_t *
1136 isp_find_ntpd(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag_id, uint32_t seq_id)
1137 {
1138         inot_private_data_t *ntp;
1139         for (ntp = tptr->ntpool; ntp < &tptr->ntpool[ATPDPSIZE]; ntp++) {
1140                 if (ntp->rd.tag_id == tag_id && ntp->rd.seq_id == seq_id) {
1141                         return (ntp);
1142                 }
1143         }
1144         return (NULL);
1145 }
1146
1147 static ISP_INLINE void
1148 isp_put_ntpd(ispsoftc_t *isp, tstate_t *tptr, inot_private_data_t *ntp)
1149 {
1150         ntp->rd.tag_id = ntp->rd.seq_id = 0;
1151         ntp->next = tptr->ntfree;
1152         tptr->ntfree = ntp;
1153 }
1154
1155 static cam_status
1156 create_lun_state(ispsoftc_t *isp, int bus, struct cam_path *path, tstate_t **rslt)
1157 {
1158         cam_status status;
1159         lun_id_t lun;
1160         struct tslist *lhp;
1161         tstate_t *tptr;
1162         int i;
1163
1164         lun = xpt_path_lun_id(path);
1165         if (lun != CAM_LUN_WILDCARD) {
1166                 if (lun >= ISP_MAX_LUNS(isp)) {
1167                         return (CAM_LUN_INVALID);
1168                 }
1169         }
1170         if (is_lun_enabled(isp, bus, lun)) {
1171                 return (CAM_LUN_ALRDY_ENA);
1172         }
1173         tptr = malloc(sizeof (tstate_t), M_DEVBUF, M_NOWAIT|M_ZERO);
1174         if (tptr == NULL) {
1175                 return (CAM_RESRC_UNAVAIL);
1176         }
1177         tptr->ts_lun = lun;
1178         status = xpt_create_path(&tptr->owner, NULL, xpt_path_path_id(path), xpt_path_target_id(path), lun);
1179         if (status != CAM_REQ_CMP) {
1180                 free(tptr, M_DEVBUF);
1181                 return (status);
1182         }
1183         SLIST_INIT(&tptr->atios);
1184         SLIST_INIT(&tptr->inots);
1185         TAILQ_INIT(&tptr->waitq);
1186         LIST_INIT(&tptr->atfree);
1187         for (i = ATPDPSIZE-1; i >= 0; i--)
1188                 LIST_INSERT_HEAD(&tptr->atfree, &tptr->atpool[i], next);
1189         for (i = 0; i < ATPDPHASHSIZE; i++)
1190                 LIST_INIT(&tptr->atused[i]);
1191         for (i = 0; i < ATPDPSIZE-1; i++)
1192                 tptr->ntpool[i].next = &tptr->ntpool[i+1];
1193         tptr->ntfree = tptr->ntpool;
1194         tptr->hold = 1;
1195         ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp);
1196         SLIST_INSERT_HEAD(lhp, tptr, next);
1197         *rslt = tptr;
1198         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, path, "created tstate\n");
1199         return (CAM_REQ_CMP);
1200 }
1201
1202 static ISP_INLINE void
1203 destroy_lun_state(ispsoftc_t *isp, tstate_t *tptr)
1204 {
1205         union ccb *ccb;
1206         struct tslist *lhp;
1207
1208         KASSERT((tptr->hold != 0), ("tptr is not held"));
1209         KASSERT((tptr->hold == 1), ("tptr still held (%d)", tptr->hold));
1210         do {
1211                 ccb = (union ccb *)SLIST_FIRST(&tptr->atios);
1212                 if (ccb) {
1213                         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
1214                         ccb->ccb_h.status = CAM_REQ_ABORTED;
1215                         xpt_done(ccb);
1216                 }
1217         } while (ccb);
1218         do {
1219                 ccb = (union ccb *)SLIST_FIRST(&tptr->inots);
1220                 if (ccb) {
1221                         SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle);
1222                         ccb->ccb_h.status = CAM_REQ_ABORTED;
1223                         xpt_done(ccb);
1224                 }
1225         } while (ccb);
1226         ISP_GET_PC_ADDR(isp, cam_sim_bus(xpt_path_sim(tptr->owner)), lun_hash[LUN_HASH_FUNC(tptr->ts_lun)], lhp);
1227         SLIST_REMOVE(lhp, tptr, tstate, next);
1228         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, tptr->owner, "destroyed tstate\n");
1229         xpt_free_path(tptr->owner);
1230         free(tptr, M_DEVBUF);
1231 }
1232
1233 /*
1234  * Enable a lun.
1235  */
1236 static void
1237 isp_enable_lun(ispsoftc_t *isp, union ccb *ccb)
1238 {
1239         tstate_t *tptr = NULL;
1240         int bus, tm_enabled, target_role;
1241         target_id_t target;
1242         lun_id_t lun;
1243
1244
1245         /*
1246          * We only support either a wildcard target/lun or a target ID of zero and a non-wildcard lun
1247          */
1248         bus = XS_CHANNEL(ccb);
1249         target = ccb->ccb_h.target_id;
1250         lun = ccb->ccb_h.target_lun;
1251         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0|ISP_LOGCONFIG, ccb->ccb_h.path, "enabling lun %u\n", lun);
1252         if (target != CAM_TARGET_WILDCARD && target != 0) {
1253                 ccb->ccb_h.status = CAM_TID_INVALID;
1254                 xpt_done(ccb);
1255                 return;
1256         }
1257         if (target == CAM_TARGET_WILDCARD && lun != CAM_LUN_WILDCARD) {
1258                 ccb->ccb_h.status = CAM_LUN_INVALID;
1259                 xpt_done(ccb);
1260                 return;
1261         }
1262
1263         if (target != CAM_TARGET_WILDCARD && lun == CAM_LUN_WILDCARD) {
1264                 ccb->ccb_h.status = CAM_LUN_INVALID;
1265                 xpt_done(ccb);
1266                 return;
1267         }
1268         if (isp->isp_dblev & ISP_LOGTDEBUG0) {
1269                 xpt_print(ccb->ccb_h.path, "enabling lun 0x%x on channel %d\n", lun, bus);
1270         }
1271
1272         /*
1273          * Wait until we're not busy with the lun enables subsystem
1274          */
1275         isp_tmlock(isp, "isp_enable_lun");
1276
1277         /*
1278          * This is as a good a place as any to check f/w capabilities.
1279          */
1280
1281         if (IS_FC(isp)) {
1282                 if (ISP_CAP_TMODE(isp) == 0) {
1283                         xpt_print(ccb->ccb_h.path, "firmware does not support target mode\n");
1284                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
1285                         goto done;
1286                 }
1287                 /*
1288                  * We *could* handle non-SCCLUN f/w, but we'd have to
1289                  * dork with our already fragile enable/disable code.
1290                  */
1291                 if (ISP_CAP_SCCFW(isp) == 0) {
1292                         xpt_print(ccb->ccb_h.path, "firmware not SCCLUN capable\n");
1293                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
1294                         goto done;
1295                 }
1296
1297                 target_role = (FCPARAM(isp, bus)->role & ISP_ROLE_TARGET) != 0;
1298
1299         } else {
1300                 target_role = (SDPARAM(isp, bus)->role & ISP_ROLE_TARGET) != 0;
1301         }
1302
1303         /*
1304          * Create the state pointer.
1305          * It should not already exist.
1306          */
1307         tptr = get_lun_statep(isp, bus, lun);
1308         if (tptr) {
1309                 ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
1310                 goto done;
1311         }
1312         ccb->ccb_h.status = create_lun_state(isp, bus, ccb->ccb_h.path, &tptr);
1313         if (ccb->ccb_h.status != CAM_REQ_CMP) {
1314                 goto done;
1315         }
1316
1317         /*
1318          * We have a tricky maneuver to perform here.
1319          *
1320          * If target mode isn't already enabled here,
1321          * *and* our current role includes target mode,
1322          * we enable target mode here.
1323          *
1324          */
1325         ISP_GET_PC(isp, bus, tm_enabled, tm_enabled);
1326         if (tm_enabled == 0 && target_role != 0) {
1327                 if (isp_enable_target_mode(isp, bus)) {
1328                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1329                         destroy_lun_state(isp, tptr);
1330                         tptr = NULL;
1331                         goto done;
1332                 }
1333                 tm_enabled = 1;
1334         }
1335
1336         /*
1337          * Now check to see whether this bus is in target mode already.
1338          *
1339          * If not, a later role change into target mode will finish the job.
1340          */
1341         if (tm_enabled == 0) {
1342                 ISP_SET_PC(isp, bus, tm_enable_defer, 1);
1343                 ccb->ccb_h.status = CAM_REQ_CMP;
1344                 xpt_print(ccb->ccb_h.path, "Target Mode not enabled yet- lun enable deferred\n");
1345                 goto done1;
1346         }
1347
1348         /*
1349          * Enable the lun.
1350          */
1351         ccb->ccb_h.status = isp_enable_deferred(isp, bus, lun);
1352
1353 done:
1354         if (ccb->ccb_h.status != CAM_REQ_CMP)  {
1355                 if (tptr) {
1356                         destroy_lun_state(isp, tptr);
1357                         tptr = NULL;
1358                 }
1359         } else {
1360                 tptr->enabled = 1;
1361         }
1362 done1:
1363         if (tptr) {
1364                 rls_lun_statep(isp, tptr);
1365         }
1366
1367         /*
1368          * And we're outta here....
1369          */
1370         isp_tmunlk(isp);
1371         xpt_done(ccb);
1372 }
1373
1374 static cam_status
1375 isp_enable_deferred_luns(ispsoftc_t *isp, int bus)
1376 {
1377         tstate_t *tptr = NULL;
1378         struct tslist *lhp;
1379         int i, n;
1380
1381
1382         ISP_GET_PC(isp, bus, tm_enabled, i);
1383         if (i == 1) {
1384                 return (CAM_REQ_CMP);
1385         }
1386         ISP_GET_PC(isp, bus, tm_enable_defer, i);
1387         if (i == 0) {
1388                 return (CAM_REQ_CMP);
1389         }
1390         /*
1391          * If this succeeds, it will set tm_enable
1392          */
1393         if (isp_enable_target_mode(isp, bus)) {
1394                 return (CAM_REQ_CMP_ERR);
1395         }
1396         isp_tmlock(isp, "isp_enable_deferred_luns");
1397         for (n = i = 0; i < LUN_HASH_SIZE; i++) {
1398                 ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
1399                 SLIST_FOREACH(tptr, lhp, next) {
1400                         tptr->hold++;
1401                         if (tptr->enabled == 0) {
1402                                 if (isp_enable_deferred(isp, bus, tptr->ts_lun) == CAM_REQ_CMP) {
1403                                         tptr->enabled = 1;
1404                                         n++;
1405                                 }
1406                         } else {
1407                                 n++;
1408                         }
1409                         tptr->hold--;
1410                 }
1411         }
1412         isp_tmunlk(isp);
1413         if (n == 0) {
1414                 return (CAM_REQ_CMP_ERR);
1415         }
1416         ISP_SET_PC(isp, bus, tm_enable_defer, 0);
1417         return (CAM_REQ_CMP);
1418 }
1419
1420 static cam_status
1421 isp_enable_deferred(ispsoftc_t *isp, int bus, lun_id_t lun)
1422 {
1423         cam_status status;
1424         int luns_already_enabled;
1425
1426         ISP_GET_PC(isp, bus, tm_luns_enabled, luns_already_enabled);
1427         isp_prt(isp, ISP_LOGTINFO, "%s: bus %d lun %u luns_enabled %d", __func__, bus, lun, luns_already_enabled);
1428         if (IS_24XX(isp) || (IS_FC(isp) && luns_already_enabled)) {
1429                 status = CAM_REQ_CMP;
1430         } else {
1431                 int cmd_cnt, not_cnt;
1432
1433                 if (IS_23XX(isp)) {
1434                         cmd_cnt = DFLT_CMND_CNT;
1435                         not_cnt = DFLT_INOT_CNT;
1436                 } else {
1437                         cmd_cnt = 64;
1438                         not_cnt = 8;
1439                 }
1440                 status = CAM_REQ_INPROG;
1441                 isp->isp_osinfo.rptr = &status;
1442                 if (isp_lun_cmd(isp, RQSTYPE_ENABLE_LUN, bus, lun == CAM_LUN_WILDCARD? 0 : lun, cmd_cnt, not_cnt)) {
1443                         status = CAM_RESRC_UNAVAIL;
1444                 } else {
1445                         mtx_sleep(&status, &isp->isp_lock, PRIBIO, "isp_enable_deferred", 0);
1446                 }
1447                 isp->isp_osinfo.rptr = NULL;
1448         }
1449         if (status == CAM_REQ_CMP) {
1450                 ISP_SET_PC(isp, bus, tm_luns_enabled, 1);
1451                 isp_prt(isp, ISP_LOGCONFIG|ISP_LOGTINFO, "bus %d lun %u now enabled for target mode", bus, lun);
1452         }
1453         return (status);
1454 }
1455
1456 static void
1457 isp_disable_lun(ispsoftc_t *isp, union ccb *ccb)
1458 {
1459         tstate_t *tptr = NULL;
1460         int bus;
1461         cam_status status;
1462         target_id_t target;
1463         lun_id_t lun;
1464
1465         bus = XS_CHANNEL(ccb);
1466         target = ccb->ccb_h.target_id;
1467         lun = ccb->ccb_h.target_lun;
1468         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0|ISP_LOGCONFIG, ccb->ccb_h.path, "disabling lun %u\n", lun);
1469         if (target != CAM_TARGET_WILDCARD && target != 0) {
1470                 ccb->ccb_h.status = CAM_TID_INVALID;
1471                 xpt_done(ccb);
1472                 return;
1473         }
1474
1475         if (target == CAM_TARGET_WILDCARD && lun != CAM_LUN_WILDCARD) {
1476                 ccb->ccb_h.status = CAM_LUN_INVALID;
1477                 xpt_done(ccb);
1478                 return;
1479         }
1480
1481         if (target != CAM_TARGET_WILDCARD && lun == CAM_LUN_WILDCARD) {
1482                 ccb->ccb_h.status = CAM_LUN_INVALID;
1483                 xpt_done(ccb);
1484                 return;
1485         }
1486
1487         /*
1488          * See if we're busy disabling a lun now.
1489          */
1490         isp_tmlock(isp, "isp_disable_lun");
1491         status = CAM_REQ_INPROG;
1492
1493         /*
1494          * Find the state pointer.
1495          */
1496         if ((tptr = get_lun_statep(isp, bus, lun)) == NULL) {
1497                 status = CAM_PATH_INVALID;
1498                 goto done;
1499         }
1500
1501         /*
1502          * If we're a 24XX card, we're done.
1503          */
1504         if (IS_24XX(isp)) {
1505                 status = CAM_REQ_CMP;
1506                 goto done;
1507         }
1508
1509         /*
1510          * For SCC FW, we only deal with lun zero.
1511          */
1512         if (IS_FC(isp) && lun > 0) {
1513                 status = CAM_REQ_CMP;
1514                 goto done;
1515         }
1516         isp->isp_osinfo.rptr = &status;
1517         if (isp_lun_cmd(isp, RQSTYPE_ENABLE_LUN, bus, lun, 0, 0)) {
1518                 status = CAM_RESRC_UNAVAIL;
1519         } else {
1520                 mtx_sleep(ccb, &isp->isp_lock, PRIBIO, "isp_disable_lun", 0);
1521         }
1522         isp->isp_osinfo.rptr = NULL;
1523 done:
1524         if (status == CAM_REQ_CMP) {
1525                 tptr->enabled = 0;
1526                 /*
1527                  * If we have no more luns enabled for this bus,
1528                  * delete all tracked wwns for it (if we are FC),
1529                  * and disable target mode.
1530                  */
1531                 if (is_any_lun_enabled(isp, bus) == 0) {
1532                         isp_del_all_wwn_entries(isp, bus);
1533                         if (isp_disable_target_mode(isp, bus)) {
1534                                 status = CAM_REQ_CMP_ERR;
1535                         }
1536                 }
1537         }
1538         ccb->ccb_h.status = status;
1539         if (status == CAM_REQ_CMP) {
1540                 destroy_lun_state(isp, tptr);
1541                 xpt_print(ccb->ccb_h.path, "lun now disabled for target mode\n");
1542         } else {
1543                 if (tptr)
1544                         rls_lun_statep(isp, tptr);
1545         }
1546         isp_tmunlk(isp);
1547         xpt_done(ccb);
1548 }
1549
1550 static int
1551 isp_enable_target_mode(ispsoftc_t *isp, int bus)
1552 {
1553         int tm_enabled;
1554
1555         ISP_GET_PC(isp, bus, tm_enabled, tm_enabled);
1556         if (tm_enabled != 0) {
1557                 return (0);
1558         }
1559         if (IS_SCSI(isp)) {
1560                 mbreg_t mbs;
1561                 MBSINIT(&mbs, MBOX_ENABLE_TARGET_MODE, MBLOGALL, 0);
1562                 mbs.param[0] = MBOX_ENABLE_TARGET_MODE;
1563                 mbs.param[1] = ENABLE_TARGET_FLAG|ENABLE_TQING_FLAG;
1564                 mbs.param[2] = bus << 7;
1565                 if (isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs) < 0 || mbs.param[0] != MBOX_COMMAND_COMPLETE) {
1566                         isp_prt(isp, ISP_LOGERR, "Unable to enable Target Role on Bus %d", bus);
1567                         return (EIO);
1568                 }
1569         }
1570         ISP_SET_PC(isp, bus, tm_enabled, 1);
1571         isp_prt(isp, ISP_LOGINFO, "Target Role enabled on Bus %d", bus);
1572         return (0);
1573 }
1574
1575 static int
1576 isp_disable_target_mode(ispsoftc_t *isp, int bus)
1577 {
1578         int tm_enabled;
1579
1580         ISP_GET_PC(isp, bus, tm_enabled, tm_enabled);
1581         if (tm_enabled == 0) {
1582                 return (0);
1583         }
1584         if (IS_SCSI(isp)) {
1585                 mbreg_t mbs;
1586                 MBSINIT(&mbs, MBOX_ENABLE_TARGET_MODE, MBLOGALL, 0);
1587                 mbs.param[2] = bus << 7;
1588                 if (isp_control(isp, ISPCTL_RUN_MBOXCMD, &mbs) < 0 || mbs.param[0] != MBOX_COMMAND_COMPLETE) {
1589                         isp_prt(isp, ISP_LOGERR, "Unable to disable Target Role on Bus %d", bus);
1590                         return (EIO);
1591                 }
1592         }
1593         ISP_SET_PC(isp, bus, tm_enabled, 0);
1594         isp_prt(isp, ISP_LOGINFO, "Target Role disabled on Bus %d", bus);
1595         return (0);
1596 }
1597
1598 static void
1599 isp_ledone(ispsoftc_t *isp, lun_entry_t *lep)
1600 {
1601         uint32_t *rptr;
1602
1603         rptr = isp->isp_osinfo.rptr;
1604         if (lep->le_status != LUN_OK) {
1605                 isp_prt(isp, ISP_LOGERR, "ENABLE/MODIFY LUN returned 0x%x", lep->le_status);
1606                 if (rptr) {
1607                         *rptr = CAM_REQ_CMP_ERR;
1608                         wakeup_one(rptr);
1609                 }
1610         } else {
1611                 if (rptr) {
1612                         *rptr = CAM_REQ_CMP;
1613                         wakeup_one(rptr);
1614                 }
1615         }
1616 }
1617
1618 static void
1619 isp_target_start_ctio(ispsoftc_t *isp, union ccb *ccb, enum Start_Ctio_How how)
1620 {
1621         int fctape, sendstatus, resid;
1622         tstate_t *tptr;
1623         fcparam *fcp;
1624         atio_private_data_t *atp;
1625         struct ccb_scsiio *cso;
1626         uint32_t dmaresult, handle, xfrlen, sense_length, tmp;
1627         uint8_t local[QENTRY_LEN];
1628
1629         tptr = get_lun_statep(isp, XS_CHANNEL(ccb), XS_LUN(ccb));
1630         if (tptr == NULL) {
1631                 tptr = get_lun_statep(isp, XS_CHANNEL(ccb), CAM_LUN_WILDCARD);
1632                 if (tptr == NULL) {
1633                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] cannot find tstate pointer", __func__, ccb->csio.tag_id);
1634                         ccb->ccb_h.status = CAM_DEV_NOT_THERE;
1635                         xpt_done(ccb);
1636                         return;
1637                 }
1638         }
1639         isp_prt(isp, ISP_LOGTDEBUG0, "%s: ENTRY[0x%x] how %u xfrlen %u sendstatus %d sense_len %u", __func__, ccb->csio.tag_id, how, ccb->csio.dxfer_len,
1640             (ccb->ccb_h.flags & CAM_SEND_STATUS) != 0, ((ccb->ccb_h.flags & CAM_SEND_SENSE)? ccb->csio.sense_len : 0));
1641
1642         switch (how) {
1643         case FROM_TIMER:
1644         case FROM_CAM:
1645                 /*
1646                  * Insert at the tail of the list, if any, waiting CTIO CCBs
1647                  */
1648                 TAILQ_INSERT_TAIL(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
1649                 break;
1650         case FROM_SRR:
1651         case FROM_CTIO_DONE:
1652                 TAILQ_INSERT_HEAD(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
1653                 break;
1654         }
1655
1656         while (TAILQ_FIRST(&tptr->waitq) != NULL) {
1657                 ccb = (union ccb *) TAILQ_FIRST(&tptr->waitq);
1658                 TAILQ_REMOVE(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
1659
1660                 cso = &ccb->csio;
1661                 xfrlen = cso->dxfer_len;
1662                 if (xfrlen == 0) {
1663                         if ((ccb->ccb_h.flags & CAM_SEND_STATUS) == 0) {
1664                                 ISP_PATH_PRT(isp, ISP_LOGERR, ccb->ccb_h.path, "a data transfer length of zero but no status to send is wrong\n");
1665                                 ccb->ccb_h.status = CAM_REQ_INVALID;
1666                                 xpt_done(ccb);
1667                                 continue;
1668                         }
1669                 }
1670
1671                 atp = isp_find_atpd(isp, tptr, cso->tag_id);
1672                 if (atp == NULL) {
1673                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] cannot find private data adjunct in %s", __func__, cso->tag_id, __func__);
1674                         isp_dump_atpd(isp, tptr);
1675                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1676                         xpt_done(ccb);
1677                         continue;
1678                 }
1679
1680                 /*
1681                  * Is this command a dead duck?
1682                  */
1683                 if (atp->dead) {
1684                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] not sending a CTIO for a dead command", __func__, cso->tag_id);
1685                         ccb->ccb_h.status = CAM_REQ_ABORTED;
1686                         xpt_done(ccb);
1687                         continue;
1688                 }
1689
1690                 /*
1691                  * Check to make sure we're still in target mode.
1692                  */
1693                 fcp = FCPARAM(isp, XS_CHANNEL(ccb));
1694                 if ((fcp->role & ISP_ROLE_TARGET) == 0) {
1695                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] stopping sending a CTIO because we're no longer in target mode", __func__, cso->tag_id);
1696                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1697                         xpt_done(ccb);
1698                         continue;
1699                 }
1700
1701                 /*
1702                  * We're only handling ATPD_CCB_OUTSTANDING outstanding CCB at a time (one of which
1703                  * could be split into two CTIOs to split data and status).
1704                  */
1705                 if (atp->ctcnt >= ATPD_CCB_OUTSTANDING) {
1706                         isp_prt(isp, ISP_LOGTINFO, "[0x%x] handling only %d CCBs at a time (flags for this ccb: 0x%x)", cso->tag_id, ATPD_CCB_OUTSTANDING, ccb->ccb_h.flags);
1707                         TAILQ_INSERT_HEAD(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
1708                         break;
1709                 }
1710
1711                 /*
1712                  * Does the initiator expect FC-Tape style responses?
1713                  */
1714                 if ((atp->word3 & PRLI_WD3_RETRY) && fcp->fctape_enabled) {
1715                         fctape = 1;
1716                 } else {
1717                         fctape = 0;
1718                 }
1719
1720                 /*
1721                  * If we already did the data xfer portion of a CTIO that sends data
1722                  * and status, don't do it again and do the status portion now.
1723                  */
1724                 if (atp->sendst) {
1725                         isp_prt(isp, ISP_LOGTINFO, "[0x%x] now sending synthesized status orig_dl=%u xfered=%u bit=%u",
1726                             cso->tag_id, atp->orig_datalen, atp->bytes_xfered, atp->bytes_in_transit);
1727                         xfrlen = 0;     /* we already did the data transfer */
1728                         atp->sendst = 0;
1729                 }
1730                 if (ccb->ccb_h.flags & CAM_SEND_STATUS) {
1731                         sendstatus = 1;
1732                 } else {
1733                         sendstatus = 0;
1734                 }
1735
1736                 if (ccb->ccb_h.flags & CAM_SEND_SENSE) {
1737                         KASSERT((sendstatus != 0), ("how can you have CAM_SEND_SENSE w/o CAM_SEND_STATUS?"));
1738                         /*
1739                          * Sense length is not the entire sense data structure size. Periph
1740                          * drivers don't seem to be setting sense_len to reflect the actual
1741                          * size. We'll peek inside to get the right amount.
1742                          */
1743                         sense_length = cso->sense_len;
1744
1745                         /*
1746                          * This 'cannot' happen
1747                          */
1748                         if (sense_length > (XCMD_SIZE - MIN_FCP_RESPONSE_SIZE)) {
1749                                 sense_length = XCMD_SIZE - MIN_FCP_RESPONSE_SIZE;
1750                         }
1751                 } else {
1752                         sense_length = 0;
1753                 }
1754
1755                 memset(local, 0, QENTRY_LEN);
1756
1757                 /*
1758                  * Check for overflow
1759                  */
1760                 tmp = atp->bytes_xfered + atp->bytes_in_transit + xfrlen;
1761                 if (tmp > atp->orig_datalen) {
1762                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] data overflow by %u bytes", __func__, cso->tag_id, tmp - atp->orig_datalen);
1763                         ccb->ccb_h.status = CAM_DATA_RUN_ERR;
1764                         xpt_done(ccb);
1765                         continue;
1766                 }
1767
1768                 if (IS_24XX(isp)) {
1769                         ct7_entry_t *cto = (ct7_entry_t *) local;
1770
1771                         cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
1772                         cto->ct_header.rqs_entry_count = 1;
1773                         cto->ct_header.rqs_seqno |= ATPD_SEQ_NOTIFY_CAM;
1774                         ATPD_SET_SEQNO(cto, atp);
1775                         cto->ct_nphdl = atp->nphdl;
1776                         cto->ct_rxid = atp->tag;
1777                         cto->ct_iid_lo = atp->portid;
1778                         cto->ct_iid_hi = atp->portid >> 16;
1779                         cto->ct_oxid = atp->oxid;
1780                         cto->ct_vpidx = ISP_GET_VPIDX(isp, XS_CHANNEL(ccb));
1781                         cto->ct_timeout = 120;
1782                         cto->ct_flags = atp->tattr << CT7_TASK_ATTR_SHIFT;
1783
1784                         /*
1785                          * Mode 1, status, no data. Only possible when we are sending status, have
1786                          * no data to transfer, and any sense data can fit into a ct7_entry_t.
1787                          *
1788                          * Mode 2, status, no data. We have to use this in the case that
1789                          * the sense data won't fit into a ct7_entry_t.
1790                          *
1791                          */
1792                         if (sendstatus && xfrlen == 0) {
1793                                 cto->ct_flags |= CT7_SENDSTATUS | CT7_NO_DATA;
1794                                 resid = atp->orig_datalen - atp->bytes_xfered - atp->bytes_in_transit;
1795                                 if (sense_length <= MAXRESPLEN_24XX) {
1796                                         if (resid < 0) {
1797                                                 cto->ct_resid = -resid;
1798                                         } else if (resid > 0) {
1799                                                 cto->ct_resid = resid;
1800                                         }
1801                                         cto->ct_flags |= CT7_FLAG_MODE1;
1802                                         cto->ct_scsi_status = cso->scsi_status;
1803                                         if (resid < 0) {
1804                                                 cto->ct_scsi_status |= (FCP_RESID_OVERFLOW << 8);
1805                                         } else if (resid > 0) {
1806                                                 cto->ct_scsi_status |= (FCP_RESID_UNDERFLOW << 8);
1807                                         }
1808                                         if (fctape) {
1809                                                 cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF;
1810                                         }
1811                                         if (sense_length) {
1812                                                 cto->ct_scsi_status |= (FCP_SNSLEN_VALID << 8);
1813                                                 cto->rsp.m1.ct_resplen = cto->ct_senselen = sense_length;
1814                                                 memcpy(cto->rsp.m1.ct_resp, &cso->sense_data, sense_length);
1815                                         }
1816                                 } else {
1817                                         bus_addr_t addr;
1818                                         char buf[XCMD_SIZE];
1819                                         fcp_rsp_iu_t *rp;
1820
1821                                         if (atp->ests == NULL) {
1822                                                 atp->ests = isp_get_ecmd(isp);
1823                                                 if (atp->ests == NULL) {
1824                                                         TAILQ_INSERT_HEAD(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
1825                                                         break;
1826                                                 }
1827                                         }
1828                                         memset(buf, 0, sizeof (buf));
1829                                         rp = (fcp_rsp_iu_t *)buf;
1830                                         if (fctape) {
1831                                                 cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF;
1832                                                 rp->fcp_rsp_bits |= FCP_CONF_REQ;
1833                                         }
1834                                         cto->ct_flags |= CT7_FLAG_MODE2;
1835                                         rp->fcp_rsp_scsi_status = cso->scsi_status;
1836                                         if (resid < 0) {
1837                                                 rp->fcp_rsp_resid = -resid;
1838                                                 rp->fcp_rsp_bits |= FCP_RESID_OVERFLOW;
1839                                         } else if (resid > 0) {
1840                                                 rp->fcp_rsp_resid = resid;
1841                                                 rp->fcp_rsp_bits |= FCP_RESID_UNDERFLOW;
1842                                         }
1843                                         if (sense_length) {
1844                                                 rp->fcp_rsp_snslen = sense_length;
1845                                                 cto->ct_senselen = sense_length;
1846                                                 rp->fcp_rsp_bits |= FCP_SNSLEN_VALID;
1847                                                 isp_put_fcp_rsp_iu(isp, rp, atp->ests);
1848                                                 memcpy(((fcp_rsp_iu_t *)atp->ests)->fcp_rsp_extra, &cso->sense_data, sense_length);
1849                                         } else {
1850                                                 isp_put_fcp_rsp_iu(isp, rp, atp->ests);
1851                                         }
1852                                         if (isp->isp_dblev & ISP_LOGTDEBUG1) {
1853                                                 isp_print_bytes(isp, "FCP Response Frame After Swizzling", MIN_FCP_RESPONSE_SIZE + sense_length, atp->ests);
1854                                         }
1855                                         addr = isp->isp_osinfo.ecmd_dma;
1856                                         addr += ((((isp_ecmd_t *)atp->ests) - isp->isp_osinfo.ecmd_base) * XCMD_SIZE);
1857                                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: ests base %p vaddr %p ecmd_dma %jx addr %jx len %u", __func__, isp->isp_osinfo.ecmd_base, atp->ests,
1858                                             (uintmax_t) isp->isp_osinfo.ecmd_dma, (uintmax_t)addr, MIN_FCP_RESPONSE_SIZE + sense_length);
1859                                         cto->rsp.m2.ct_datalen = MIN_FCP_RESPONSE_SIZE + sense_length;
1860                                         cto->rsp.m2.ct_fcp_rsp_iudata.ds_base = DMA_LO32(addr);
1861                                         cto->rsp.m2.ct_fcp_rsp_iudata.ds_basehi = DMA_HI32(addr);
1862                                         cto->rsp.m2.ct_fcp_rsp_iudata.ds_count = MIN_FCP_RESPONSE_SIZE + sense_length;
1863                                 }
1864                                 if (sense_length) {
1865                                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d slen %u sense: %x %x/%x/%x", __func__,
1866                                             cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, cto->ct_resid, sense_length,
1867                                             cso->sense_data.error_code, cso->sense_data.sense_buf[1], cso->sense_data.sense_buf[11], cso->sense_data.sense_buf[12]);
1868                                 } else {
1869                                         isp_prt(isp, ISP_LOGDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d", __func__,
1870                                             cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, cto->ct_resid);
1871                                 }
1872                                 atp->state = ATPD_STATE_LAST_CTIO;
1873                         }
1874
1875                         /*
1876                          * Mode 0 data transfers, *possibly* with status.
1877                          */
1878                         if (xfrlen != 0) {
1879                                 cto->ct_flags |= CT7_FLAG_MODE0;
1880                                 if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1881                                         cto->ct_flags |= CT7_DATA_IN;
1882                                 } else {
1883                                         cto->ct_flags |= CT7_DATA_OUT;
1884                                 }
1885
1886                                 cto->rsp.m0.reloff = atp->bytes_xfered + atp->bytes_in_transit;
1887                                 cto->rsp.m0.ct_xfrlen = xfrlen;
1888
1889 #ifdef  DEBUG
1890                                 if (ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame && xfrlen > ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame) {
1891                                         isp_prt(isp, ISP_LOGWARN, "%s: truncating data frame with xfrlen %d to %d", __func__, xfrlen, xfrlen - (xfrlen >> 2));
1892                                         ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame = 0;
1893                                         cto->rsp.m0.ct_xfrlen -= xfrlen >> 2;
1894                                 }
1895 #endif
1896                                 if (sendstatus) {
1897                                         resid = atp->orig_datalen - atp->bytes_xfered - xfrlen;
1898                                         if (cso->scsi_status == SCSI_STATUS_OK && resid == 0 /* && fctape == 0 */) {
1899                                                 cto->ct_flags |= CT7_SENDSTATUS;
1900                                                 atp->state = ATPD_STATE_LAST_CTIO;
1901                                                 if (fctape) {
1902                                                         cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF;
1903                                                 }
1904                                         } else {
1905                                                 atp->sendst = 1;        /* send status later */
1906                                                 cto->ct_header.rqs_seqno &= ~ATPD_SEQ_NOTIFY_CAM;
1907                                                 atp->state = ATPD_STATE_CTIO;
1908                                         }
1909                                 } else {
1910                                         atp->state = ATPD_STATE_CTIO;
1911                                 }
1912                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x xfrlen=%u off=%u", __func__,
1913                                     cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, xfrlen, atp->bytes_xfered);
1914                         }
1915                 } else if (IS_FC(isp)) {
1916                         ct2_entry_t *cto = (ct2_entry_t *) local;
1917
1918                         if (isp->isp_osinfo.sixtyfourbit)
1919                                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO3;
1920                         else
1921                                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO2;
1922                         cto->ct_header.rqs_entry_count = 1;
1923                         cto->ct_header.rqs_seqno |= ATPD_SEQ_NOTIFY_CAM;
1924                         ATPD_SET_SEQNO(cto, atp);
1925                         if (ISP_CAP_2KLOGIN(isp) == 0) {
1926                                 ((ct2e_entry_t *)cto)->ct_iid = cso->init_id;
1927                         } else {
1928                                 cto->ct_iid = cso->init_id;
1929                                 if (ISP_CAP_SCCFW(isp) == 0) {
1930                                         cto->ct_lun = ccb->ccb_h.target_lun;
1931                                 }
1932                         }
1933                         cto->ct_timeout = 10;
1934                         cto->ct_rxid = cso->tag_id;
1935
1936                         /*
1937                          * Mode 1, status, no data. Only possible when we are sending status, have
1938                          * no data to transfer, and the sense length can fit in the ct7_entry.
1939                          *
1940                          * Mode 2, status, no data. We have to use this in the case the response
1941                          * length won't fit into a ct2_entry_t.
1942                          *
1943                          * We'll fill out this structure with information as if this were a
1944                          * Mode 1. The hardware layer will create the Mode 2 FCP RSP IU as
1945                          * needed based upon this.
1946                          */
1947                         if (sendstatus && xfrlen == 0) {
1948                                 cto->ct_flags |= CT2_SENDSTATUS | CT2_NO_DATA;
1949                                 resid = atp->orig_datalen - atp->bytes_xfered - atp->bytes_in_transit;
1950                                 if (sense_length <= MAXRESPLEN) {
1951                                         if (resid < 0) {
1952                                                 cto->ct_resid = -resid;
1953                                         } else if (resid > 0) {
1954                                                 cto->ct_resid = resid;
1955                                         }
1956                                         cto->ct_flags |= CT2_FLAG_MODE1;
1957                                         cto->rsp.m1.ct_scsi_status = cso->scsi_status;
1958                                         if (resid < 0) {
1959                                                 cto->rsp.m1.ct_scsi_status |= CT2_DATA_OVER;
1960                                         } else if (resid > 0) {
1961                                                 cto->rsp.m1.ct_scsi_status |= CT2_DATA_UNDER;
1962                                         }
1963                                         if (fctape) {
1964                                                 cto->ct_flags |= CT2_CONFIRM;
1965                                         }
1966                                         if (sense_length) {
1967                                                 cto->rsp.m1.ct_scsi_status |= CT2_SNSLEN_VALID;
1968                                                 cto->rsp.m1.ct_resplen = cto->rsp.m1.ct_senselen = sense_length;
1969                                                 memcpy(cto->rsp.m1.ct_resp, &cso->sense_data, sense_length);
1970                                         }
1971                                 } else {
1972                                         bus_addr_t addr;
1973                                         char buf[XCMD_SIZE];
1974                                         fcp_rsp_iu_t *rp;
1975
1976                                         if (atp->ests == NULL) {
1977                                                 atp->ests = isp_get_ecmd(isp);
1978                                                 if (atp->ests == NULL) {
1979                                                         TAILQ_INSERT_HEAD(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
1980                                                         break;
1981                                                 }
1982                                         }
1983                                         memset(buf, 0, sizeof (buf));
1984                                         rp = (fcp_rsp_iu_t *)buf;
1985                                         if (fctape) {
1986                                                 cto->ct_flags |= CT2_CONFIRM;
1987                                                 rp->fcp_rsp_bits |= FCP_CONF_REQ;
1988                                         }
1989                                         cto->ct_flags |= CT2_FLAG_MODE2;
1990                                         rp->fcp_rsp_scsi_status = cso->scsi_status;
1991                                         if (resid < 0) {
1992                                                 rp->fcp_rsp_resid = -resid;
1993                                                 rp->fcp_rsp_bits |= FCP_RESID_OVERFLOW;
1994                                         } else if (resid > 0) {
1995                                                 rp->fcp_rsp_resid = resid;
1996                                                 rp->fcp_rsp_bits |= FCP_RESID_UNDERFLOW;
1997                                         }
1998                                         if (sense_length) {
1999                                                 rp->fcp_rsp_snslen = sense_length;
2000                                                 rp->fcp_rsp_bits |= FCP_SNSLEN_VALID;
2001                                                 isp_put_fcp_rsp_iu(isp, rp, atp->ests);
2002                                                 memcpy(((fcp_rsp_iu_t *)atp->ests)->fcp_rsp_extra, &cso->sense_data, sense_length);
2003                                         } else {
2004                                                 isp_put_fcp_rsp_iu(isp, rp, atp->ests);
2005                                         }
2006                                         if (isp->isp_dblev & ISP_LOGTDEBUG1) {
2007                                                 isp_print_bytes(isp, "FCP Response Frame After Swizzling", MIN_FCP_RESPONSE_SIZE + sense_length, atp->ests);
2008                                         }
2009                                         addr = isp->isp_osinfo.ecmd_dma;
2010                                         addr += ((((isp_ecmd_t *)atp->ests) - isp->isp_osinfo.ecmd_base) * XCMD_SIZE);
2011                                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: ests base %p vaddr %p ecmd_dma %jx addr %jx len %u", __func__, isp->isp_osinfo.ecmd_base, atp->ests,
2012                                             (uintmax_t) isp->isp_osinfo.ecmd_dma, (uintmax_t)addr, MIN_FCP_RESPONSE_SIZE + sense_length);
2013                                         cto->rsp.m2.ct_datalen = MIN_FCP_RESPONSE_SIZE + sense_length;
2014                                         if (isp->isp_osinfo.sixtyfourbit) {
2015                                                 cto->rsp.m2.u.ct_fcp_rsp_iudata_64.ds_base = DMA_LO32(addr);
2016                                                 cto->rsp.m2.u.ct_fcp_rsp_iudata_64.ds_basehi = DMA_HI32(addr);
2017                                                 cto->rsp.m2.u.ct_fcp_rsp_iudata_64.ds_count = MIN_FCP_RESPONSE_SIZE + sense_length;
2018                                         } else {
2019                                                 cto->rsp.m2.u.ct_fcp_rsp_iudata_32.ds_base = DMA_LO32(addr);
2020                                                 cto->rsp.m2.u.ct_fcp_rsp_iudata_32.ds_count = MIN_FCP_RESPONSE_SIZE + sense_length;
2021                                         }
2022                                 }
2023                                 if (sense_length) {
2024                                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO2[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d sense: %x %x/%x/%x", __func__,
2025                                             cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cso->scsi_status, cto->ct_flags, cto->ct_resid,
2026                                             cso->sense_data.error_code, cso->sense_data.sense_buf[1], cso->sense_data.sense_buf[11], cso->sense_data.sense_buf[12]);
2027                                 } else {
2028                                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO2[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d", __func__, cto->ct_rxid,
2029                                             ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cso->scsi_status, cto->ct_flags, cto->ct_resid);
2030                                 }
2031                                 atp->state = ATPD_STATE_LAST_CTIO;
2032                         }
2033
2034                         if (xfrlen != 0) {
2035                                 cto->ct_flags |= CT2_FLAG_MODE0;
2036                                 if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
2037                                         cto->ct_flags |= CT2_DATA_IN;
2038                                 } else {
2039                                         cto->ct_flags |= CT2_DATA_OUT;
2040                                 }
2041
2042                                 cto->ct_reloff = atp->bytes_xfered + atp->bytes_in_transit;
2043                                 cto->rsp.m0.ct_xfrlen = xfrlen;
2044
2045                                 if (sendstatus) {
2046                                         resid = atp->orig_datalen - atp->bytes_xfered - xfrlen;
2047                                         if (cso->scsi_status == SCSI_STATUS_OK && resid == 0 /*&& fctape == 0*/) {
2048                                                 cto->ct_flags |= CT2_SENDSTATUS;
2049                                                 atp->state = ATPD_STATE_LAST_CTIO;
2050                                                 if (fctape) {
2051                                                         cto->ct_flags |= CT2_CONFIRM;
2052                                                 }
2053                                         } else {
2054                                                 atp->sendst = 1;        /* send status later */
2055                                                 cto->ct_header.rqs_seqno &= ~ATPD_SEQ_NOTIFY_CAM;
2056                                                 atp->state = ATPD_STATE_CTIO;
2057                                         }
2058                                 } else {
2059                                         atp->state = ATPD_STATE_CTIO;
2060                                 }
2061                         }
2062                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO2[%x] seq %u nc %d CDB0=%x scsi status %x flags %x resid %d xfrlen %u offset %u", __func__, cto->ct_rxid,
2063                             ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cso->scsi_status, cto->ct_flags, cto->ct_resid, cso->dxfer_len, atp->bytes_xfered);
2064                 } else {
2065                         ct_entry_t *cto = (ct_entry_t *) local;
2066
2067                         cto->ct_header.rqs_entry_type = RQSTYPE_CTIO;
2068                         cto->ct_header.rqs_entry_count = 1;
2069                         cto->ct_header.rqs_seqno |= ATPD_SEQ_NOTIFY_CAM;
2070                         ATPD_SET_SEQNO(cto, atp);
2071                         cto->ct_iid = cso->init_id;
2072                         cto->ct_iid |= XS_CHANNEL(ccb) << 7;
2073                         cto->ct_tgt = ccb->ccb_h.target_id;
2074                         cto->ct_lun = ccb->ccb_h.target_lun;
2075                         cto->ct_fwhandle = cso->tag_id;
2076                         if (atp->rxid) {
2077                                 cto->ct_tag_val = atp->rxid;
2078                                 cto->ct_flags |= CT_TQAE;
2079                         }
2080                         if (ccb->ccb_h.flags & CAM_DIS_DISCONNECT) {
2081                                 cto->ct_flags |= CT_NODISC;
2082                         }
2083                         if (cso->dxfer_len == 0) {
2084                                 cto->ct_flags |= CT_NO_DATA;
2085                         } else if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
2086                                 cto->ct_flags |= CT_DATA_IN;
2087                         } else {
2088                                 cto->ct_flags |= CT_DATA_OUT;
2089                         }
2090                         if (ccb->ccb_h.flags & CAM_SEND_STATUS) {
2091                                 cto->ct_flags |= CT_SENDSTATUS|CT_CCINCR;
2092                                 cto->ct_scsi_status = cso->scsi_status;
2093                                 cto->ct_resid = atp->orig_datalen - atp->bytes_xfered - atp->bytes_in_transit - xfrlen;
2094                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO[%x] seq %u nc %d scsi status %x resid %d tag_id %x", __func__,
2095                                     cto->ct_fwhandle, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), cso->scsi_status, cso->resid, cso->tag_id);
2096                         }
2097                         ccb->ccb_h.flags &= ~CAM_SEND_SENSE;
2098                         cto->ct_timeout = 10;
2099                 }
2100
2101                 if (isp_get_pcmd(isp, ccb)) {
2102                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "out of PCMDs\n");
2103                         TAILQ_INSERT_HEAD(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
2104                         break;
2105                 }
2106                 if (isp_allocate_xs_tgt(isp, ccb, &handle)) {
2107                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "No XFLIST pointers for %s\n", __func__);
2108                         TAILQ_INSERT_HEAD(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
2109                         isp_free_pcmd(isp, ccb);
2110                         break;
2111                 }
2112                 atp->bytes_in_transit += xfrlen;
2113                 PISP_PCMD(ccb)->datalen = xfrlen;
2114
2115
2116                 /*
2117                  * Call the dma setup routines for this entry (and any subsequent
2118                  * CTIOs) if there's data to move, and then tell the f/w it's got
2119                  * new things to play with. As with isp_start's usage of DMA setup,
2120                  * any swizzling is done in the machine dependent layer. Because
2121                  * of this, we put the request onto the queue area first in native
2122                  * format.
2123                  */
2124
2125                 if (IS_24XX(isp)) {
2126                         ct7_entry_t *cto = (ct7_entry_t *) local;
2127                         cto->ct_syshandle = handle;
2128                 } else if (IS_FC(isp)) {
2129                         ct2_entry_t *cto = (ct2_entry_t *) local;
2130                         cto->ct_syshandle = handle;
2131                 } else {
2132                         ct_entry_t *cto = (ct_entry_t *) local;
2133                         cto->ct_syshandle = handle;
2134                 }
2135
2136                 dmaresult = ISP_DMASETUP(isp, cso, (ispreq_t *) local);
2137                 if (dmaresult != CMD_QUEUED) {
2138                         isp_destroy_tgt_handle(isp, handle);
2139                         isp_free_pcmd(isp, ccb);
2140                         if (dmaresult == CMD_EAGAIN) {
2141                                 TAILQ_INSERT_HEAD(&tptr->waitq, &ccb->ccb_h, periph_links.tqe);
2142                                 break;
2143                         }
2144                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2145                         xpt_done(ccb);
2146                         continue;
2147                 }
2148                 isp->isp_nactive++;
2149                 ccb->ccb_h.status = CAM_REQ_INPROG | CAM_SIM_QUEUED;
2150                 if (xfrlen) {
2151                         ccb->ccb_h.spriv_field0 = atp->bytes_xfered;
2152                 } else {
2153                         ccb->ccb_h.spriv_field0 = ~0;
2154                 }
2155                 atp->ctcnt++;
2156                 atp->seqno++;
2157         }
2158         rls_lun_statep(isp, tptr);
2159 }
2160
2161 static void
2162 isp_refire_putback_atio(void *arg)
2163 {
2164         union ccb *ccb = arg;
2165
2166         ISP_ASSERT_LOCKED((ispsoftc_t *)XS_ISP(ccb));
2167         isp_target_putback_atio(ccb);
2168 }
2169
2170 static void
2171 isp_refire_notify_ack(void *arg)
2172 {
2173         isp_tna_t *tp  = arg;
2174         ispsoftc_t *isp = tp->isp;
2175
2176         ISP_ASSERT_LOCKED(isp);
2177         if (isp_notify_ack(isp, tp->not)) {
2178                 callout_schedule(&tp->timer, 5);
2179         } else {
2180                 free(tp, M_DEVBUF);
2181         }
2182 }
2183
2184
2185 static void
2186 isp_target_putback_atio(union ccb *ccb)
2187 {
2188         ispsoftc_t *isp;
2189         struct ccb_scsiio *cso;
2190         void *qe;
2191
2192         isp = XS_ISP(ccb);
2193
2194         qe = isp_getrqentry(isp);
2195         if (qe == NULL) {
2196                 xpt_print(ccb->ccb_h.path,
2197                     "%s: Request Queue Overflow\n", __func__);
2198                 callout_reset(&PISP_PCMD(ccb)->wdog, 10,
2199                     isp_refire_putback_atio, ccb);
2200                 return;
2201         }
2202         memset(qe, 0, QENTRY_LEN);
2203         cso = &ccb->csio;
2204         if (IS_FC(isp)) {
2205                 at2_entry_t local, *at = &local;
2206                 ISP_MEMZERO(at, sizeof (at2_entry_t));
2207                 at->at_header.rqs_entry_type = RQSTYPE_ATIO2;
2208                 at->at_header.rqs_entry_count = 1;
2209                 if (ISP_CAP_SCCFW(isp)) {
2210                         at->at_scclun = (uint16_t) ccb->ccb_h.target_lun;
2211                 } else {
2212                         at->at_lun = (uint8_t) ccb->ccb_h.target_lun;
2213                 }
2214                 at->at_status = CT_OK;
2215                 at->at_rxid = cso->tag_id;
2216                 at->at_iid = cso->ccb_h.target_id;
2217                 isp_put_atio2(isp, at, qe);
2218         } else {
2219                 at_entry_t local, *at = &local;
2220                 ISP_MEMZERO(at, sizeof (at_entry_t));
2221                 at->at_header.rqs_entry_type = RQSTYPE_ATIO;
2222                 at->at_header.rqs_entry_count = 1;
2223                 at->at_iid = cso->init_id;
2224                 at->at_iid |= XS_CHANNEL(ccb) << 7;
2225                 at->at_tgt = cso->ccb_h.target_id;
2226                 at->at_lun = cso->ccb_h.target_lun;
2227                 at->at_status = CT_OK;
2228                 at->at_tag_val = AT_GET_TAG(cso->tag_id);
2229                 at->at_handle = AT_GET_HANDLE(cso->tag_id);
2230                 isp_put_atio(isp, at, qe);
2231         }
2232         ISP_TDQE(isp, "isp_target_putback_atio", isp->isp_reqidx, qe);
2233         ISP_SYNC_REQUEST(isp);
2234         isp_complete_ctio(ccb);
2235 }
2236
2237 static void
2238 isp_complete_ctio(union ccb *ccb)
2239 {
2240         if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_INPROG) {
2241                 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
2242                 xpt_done(ccb);
2243         }
2244 }
2245
2246 /*
2247  * Handle ATIO stuff that the generic code can't.
2248  * This means handling CDBs.
2249  */
2250
2251 static void
2252 isp_handle_platform_atio(ispsoftc_t *isp, at_entry_t *aep)
2253 {
2254         tstate_t *tptr;
2255         int status, bus;
2256         struct ccb_accept_tio *atiop;
2257         atio_private_data_t *atp;
2258
2259         /*
2260          * The firmware status (except for the QLTM_SVALID bit)
2261          * indicates why this ATIO was sent to us.
2262          *
2263          * If QLTM_SVALID is set, the firmware has recommended Sense Data.
2264          *
2265          * If the DISCONNECTS DISABLED bit is set in the flags field,
2266          * we're still connected on the SCSI bus.
2267          */
2268         status = aep->at_status;
2269         if ((status & ~QLTM_SVALID) == AT_PHASE_ERROR) {
2270                 /*
2271                  * Bus Phase Sequence error. We should have sense data
2272                  * suggested by the f/w. I'm not sure quite yet what
2273                  * to do about this for CAM.
2274                  */
2275                 isp_prt(isp, ISP_LOGWARN, "PHASE ERROR");
2276                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
2277                 return;
2278         }
2279         if ((status & ~QLTM_SVALID) != AT_CDB) {
2280                 isp_prt(isp, ISP_LOGWARN, "bad atio (0x%x) leaked to platform", status);
2281                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
2282                 return;
2283         }
2284
2285         bus = GET_BUS_VAL(aep->at_iid);
2286         tptr = get_lun_statep(isp, bus, aep->at_lun);
2287         if (tptr == NULL) {
2288                 tptr = get_lun_statep(isp, bus, CAM_LUN_WILDCARD);
2289                 if (tptr == NULL) {
2290                         /*
2291                          * Because we can't autofeed sense data back with
2292                          * a command for parallel SCSI, we can't give back
2293                          * a CHECK CONDITION. We'll give back a BUSY status
2294                          * instead. This works out okay because the only
2295                          * time we should, in fact, get this, is in the
2296                          * case that somebody configured us without the
2297                          * blackhole driver, so they get what they deserve.
2298                          */
2299                         isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
2300                         return;
2301                 }
2302         }
2303
2304         atp = isp_get_atpd(isp, tptr, aep->at_handle);
2305         atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
2306         if (atiop == NULL || atp == NULL) {
2307                 /*
2308                  * Because we can't autofeed sense data back with
2309                  * a command for parallel SCSI, we can't give back
2310                  * a CHECK CONDITION. We'll give back a QUEUE FULL status
2311                  * instead. This works out okay because the only time we
2312                  * should, in fact, get this, is in the case that we've
2313                  * run out of ATIOS.
2314                  */
2315                 xpt_print(tptr->owner, "no %s for lun %d from initiator %d\n", (atp == NULL && atiop == NULL)? "ATIOs *or* ATPS" :
2316                     ((atp == NULL)? "ATPs" : "ATIOs"), aep->at_lun, aep->at_iid);
2317                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
2318                 if (atp) {
2319                         isp_put_atpd(isp, tptr, atp);
2320                 }
2321                 rls_lun_statep(isp, tptr);
2322                 return;
2323         }
2324         atp->rxid = aep->at_tag_val;
2325         atp->state = ATPD_STATE_ATIO;
2326         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
2327         tptr->atio_count--;
2328         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, atiop->ccb_h.path, "Take FREE ATIO count now %d\n", tptr->atio_count);
2329         atiop->ccb_h.target_id = aep->at_tgt;
2330         atiop->ccb_h.target_lun = aep->at_lun;
2331         if (aep->at_flags & AT_NODISC) {
2332                 atiop->ccb_h.flags |= CAM_DIS_DISCONNECT;
2333         } else {
2334                 atiop->ccb_h.flags &= ~CAM_DIS_DISCONNECT;
2335         }
2336
2337         if (status & QLTM_SVALID) {
2338                 size_t amt = ISP_MIN(QLTM_SENSELEN, sizeof (atiop->sense_data));
2339                 atiop->sense_len = amt;
2340                 ISP_MEMCPY(&atiop->sense_data, aep->at_sense, amt);
2341         } else {
2342                 atiop->sense_len = 0;
2343         }
2344
2345         atiop->init_id = GET_IID_VAL(aep->at_iid);
2346         atiop->cdb_len = aep->at_cdblen;
2347         ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cdb, aep->at_cdblen);
2348         atiop->ccb_h.status = CAM_CDB_RECVD;
2349         /*
2350          * Construct a tag 'id' based upon tag value (which may be 0..255)
2351          * and the handle (which we have to preserve).
2352          */
2353         atiop->tag_id = atp->tag;
2354         if (aep->at_flags & AT_TQAE) {
2355                 atiop->tag_action = aep->at_tag_type;
2356                 atiop->ccb_h.status |= CAM_TAG_ACTION_VALID;
2357         }
2358         atp->orig_datalen = 0;
2359         atp->bytes_xfered = 0;
2360         atp->lun = aep->at_lun;
2361         atp->nphdl = aep->at_iid;
2362         atp->portid = PORT_NONE;
2363         atp->oxid = 0;
2364         atp->cdb0 = atiop->cdb_io.cdb_bytes[0];
2365         atp->tattr = aep->at_tag_type;
2366         atp->state = ATPD_STATE_CAM;
2367         isp_prt(isp, ISP_LOGTDEBUG0, "ATIO[0x%x] CDB=0x%x lun %d", aep->at_tag_val, atp->cdb0, atp->lun);
2368         rls_lun_statep(isp, tptr);
2369 }
2370
2371 static void
2372 isp_handle_platform_atio2(ispsoftc_t *isp, at2_entry_t *aep)
2373 {
2374         lun_id_t lun;
2375         fcportdb_t *lp;
2376         tstate_t *tptr;
2377         struct ccb_accept_tio *atiop;
2378         uint16_t nphdl;
2379         atio_private_data_t *atp;
2380         inot_private_data_t *ntp;
2381
2382         /*
2383          * The firmware status (except for the QLTM_SVALID bit)
2384          * indicates why this ATIO was sent to us.
2385          *
2386          * If QLTM_SVALID is set, the firmware has recommended Sense Data.
2387          */
2388         if ((aep->at_status & ~QLTM_SVALID) != AT_CDB) {
2389                 isp_prt(isp, ISP_LOGWARN, "bogus atio (0x%x) leaked to platform", aep->at_status);
2390                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
2391                 return;
2392         }
2393
2394         if (ISP_CAP_SCCFW(isp)) {
2395                 lun = aep->at_scclun;
2396         } else {
2397                 lun = aep->at_lun;
2398         }
2399         if (ISP_CAP_2KLOGIN(isp)) {
2400                 nphdl = ((at2e_entry_t *)aep)->at_iid;
2401         } else {
2402                 nphdl = aep->at_iid;
2403         }
2404         tptr = get_lun_statep(isp, 0, lun);
2405         if (tptr == NULL) {
2406                 tptr = get_lun_statep(isp, 0, CAM_LUN_WILDCARD);
2407                 if (tptr == NULL) {
2408                         isp_prt(isp, ISP_LOGWARN, "%s: [0x%x] no state pointer for lun %d or wildcard", __func__, aep->at_rxid, lun);
2409                         if (lun == 0) {
2410                                 isp_endcmd(isp, aep, SCSI_STATUS_BUSY, 0);
2411                         } else {
2412                                 isp_endcmd(isp, aep, SCSI_STATUS_CHECK_COND | ECMD_SVALID | (0x5 << 12) | (0x25 << 16), 0);
2413                         }
2414                         return;
2415                 }
2416         }
2417
2418         /*
2419          * Start any commands pending resources first.
2420          */
2421         if (tptr->restart_queue) {
2422                 inot_private_data_t *restart_queue = tptr->restart_queue;
2423                 tptr->restart_queue = NULL;
2424                 while (restart_queue) {
2425                         ntp = restart_queue;
2426                         restart_queue = ntp->rd.nt.nt_hba;
2427                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at2_entry_t *)ntp->rd.data)->at_rxid);
2428                         isp_handle_platform_atio2(isp, (at2_entry_t *) ntp->rd.data);
2429                         isp_put_ntpd(isp, tptr, ntp);
2430                         /*
2431                          * If a recursion caused the restart queue to start to fill again,
2432                          * stop and splice the new list on top of the old list and restore
2433                          * it and go to noresrc.
2434                          */
2435                         if (tptr->restart_queue) {
2436                                 ntp = tptr->restart_queue;
2437                                 tptr->restart_queue = restart_queue;
2438                                 while (restart_queue->rd.nt.nt_hba) {
2439                                         restart_queue = restart_queue->rd.nt.nt_hba;
2440                                 }
2441                                 restart_queue->rd.nt.nt_hba = ntp;
2442                                 goto noresrc;
2443                         }
2444                 }
2445         }
2446
2447         atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
2448         if (atiop == NULL) {
2449                 goto noresrc;
2450         }
2451
2452         atp = isp_get_atpd(isp, tptr, aep->at_rxid);
2453         if (atp == NULL) {
2454                 goto noresrc;
2455         }
2456
2457         atp->state = ATPD_STATE_ATIO;
2458         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
2459         tptr->atio_count--;
2460         isp_prt(isp, ISP_LOGTDEBUG2, "Take FREE ATIO count now %d", tptr->atio_count);
2461         atiop->ccb_h.target_id = FCPARAM(isp, 0)->isp_loopid;
2462         atiop->ccb_h.target_lun = lun;
2463
2464         /*
2465          * We don't get 'suggested' sense data as we do with SCSI cards.
2466          */
2467         atiop->sense_len = 0;
2468         if (ISP_CAP_2KLOGIN(isp)) {
2469                 /*
2470                  * NB: We could not possibly have 2K logins if we
2471                  * NB: also did not have SCC FW.
2472                  */
2473                 atiop->init_id = ((at2e_entry_t *)aep)->at_iid;
2474         } else {
2475                 atiop->init_id = aep->at_iid;
2476         }
2477
2478         /*
2479          * If we're not in the port database, add ourselves.
2480          */
2481         if (!IS_2100(isp) && isp_find_pdb_by_loopid(isp, 0, atiop->init_id, &lp) == 0) {
2482                 uint64_t iid =
2483                         (((uint64_t) aep->at_wwpn[0]) << 48) |
2484                         (((uint64_t) aep->at_wwpn[1]) << 32) |
2485                         (((uint64_t) aep->at_wwpn[2]) << 16) |
2486                         (((uint64_t) aep->at_wwpn[3]) <<  0);
2487                 isp_add_wwn_entry(isp, 0, iid, atiop->init_id, PORT_ANY, 0);
2488         }
2489         atiop->cdb_len = ATIO2_CDBLEN;
2490         ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cdb, ATIO2_CDBLEN);
2491         atiop->ccb_h.status = CAM_CDB_RECVD;
2492         atiop->tag_id = atp->tag;
2493         switch (aep->at_taskflags & ATIO2_TC_ATTR_MASK) {
2494         case ATIO2_TC_ATTR_SIMPLEQ:
2495                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
2496                 atiop->tag_action = MSG_SIMPLE_Q_TAG;
2497                 break;
2498         case ATIO2_TC_ATTR_HEADOFQ:
2499                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
2500                 atiop->tag_action = MSG_HEAD_OF_Q_TAG;
2501                 break;
2502         case ATIO2_TC_ATTR_ORDERED:
2503                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
2504                 atiop->tag_action = MSG_ORDERED_Q_TAG;
2505                 break;
2506         case ATIO2_TC_ATTR_ACAQ:                /* ?? */
2507         case ATIO2_TC_ATTR_UNTAGGED:
2508         default:
2509                 atiop->tag_action = 0;
2510                 break;
2511         }
2512
2513         atp->orig_datalen = aep->at_datalen;
2514         atp->bytes_xfered = 0;
2515         atp->lun = lun;
2516         atp->nphdl = atiop->init_id;
2517         atp->sid = PORT_ANY;
2518         atp->oxid = aep->at_oxid;
2519         atp->cdb0 = aep->at_cdb[0];
2520         atp->tattr = aep->at_taskflags & ATIO2_TC_ATTR_MASK;
2521         atp->state = ATPD_STATE_CAM;
2522         xpt_done((union ccb *)atiop);
2523         isp_prt(isp, ISP_LOGTDEBUG0, "ATIO2[0x%x] CDB=0x%x lun %d datalen %u", aep->at_rxid, atp->cdb0, lun, atp->orig_datalen);
2524         rls_lun_statep(isp, tptr);
2525         return;
2526 noresrc:
2527         ntp = isp_get_ntpd(isp, tptr);
2528         if (ntp == NULL) {
2529                 rls_lun_statep(isp, tptr);
2530                 isp_endcmd(isp, aep, nphdl, 0, SCSI_STATUS_BUSY, 0);
2531                 return;
2532         }
2533         memcpy(ntp->rd.data, aep, QENTRY_LEN);
2534         ntp->rd.nt.nt_hba = tptr->restart_queue;
2535         tptr->restart_queue = ntp;
2536         rls_lun_statep(isp, tptr);
2537 }
2538
2539 static void
2540 isp_handle_platform_atio7(ispsoftc_t *isp, at7_entry_t *aep)
2541 {
2542         int cdbxlen;
2543         uint16_t lun, chan, nphdl = NIL_HANDLE;
2544         uint32_t did, sid;
2545         uint64_t wwn = INI_NONE;
2546         fcportdb_t *lp;
2547         tstate_t *tptr;
2548         struct ccb_accept_tio *atiop;
2549         atio_private_data_t *atp = NULL;
2550         atio_private_data_t *oatp;
2551         inot_private_data_t *ntp;
2552
2553         did = (aep->at_hdr.d_id[0] << 16) | (aep->at_hdr.d_id[1] << 8) | aep->at_hdr.d_id[2];
2554         sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2];
2555         lun = (aep->at_cmnd.fcp_cmnd_lun[0] << 8) | aep->at_cmnd.fcp_cmnd_lun[1];
2556
2557         /*
2558          * Find the N-port handle, and Virtual Port Index for this command.
2559          *
2560          * If we can't, we're somewhat in trouble because we can't actually respond w/o that information.
2561          * We also, as a matter of course, need to know the WWN of the initiator too.
2562          */
2563         if (ISP_CAP_MULTI_ID(isp)) {
2564                 /*
2565                  * Find the right channel based upon D_ID
2566                  */
2567                 isp_find_chan_by_did(isp, did, &chan);
2568
2569                 if (chan == ISP_NOCHAN) {
2570                         NANOTIME_T now;
2571
2572                         /*
2573                          * If we don't recognizer our own D_DID, terminate the exchange, unless we're within 2 seconds of startup
2574                          * It's a bit tricky here as we need to stash this command *somewhere*.
2575                          */
2576                         GET_NANOTIME(&now);
2577                         if (NANOTIME_SUB(&isp->isp_init_time, &now) > 2000000000ULL) {
2578                                 isp_prt(isp, ISP_LOGWARN, "%s: [RX_ID 0x%x] D_ID %x not found on any channel- dropping", __func__, aep->at_rxid, did);
2579                                 isp_endcmd(isp, aep, NIL_HANDLE, ISP_NOCHAN, ECMD_TERMINATE, 0);
2580                                 return;
2581                         }
2582                         tptr = get_lun_statep(isp, 0, 0);
2583                         if (tptr == NULL) {
2584                                 tptr = get_lun_statep(isp, 0, CAM_LUN_WILDCARD);
2585                                 if (tptr == NULL) {
2586                                         isp_prt(isp, ISP_LOGWARN, "%s: [RX_ID 0x%x] D_ID %x not found on any channel and no tptr- dropping", __func__, aep->at_rxid, did);
2587                                         isp_endcmd(isp, aep, NIL_HANDLE, ISP_NOCHAN, ECMD_TERMINATE, 0);
2588                                         return;
2589                                 }
2590                         }
2591                         isp_prt(isp, ISP_LOGWARN, "%s: [RX_ID 0x%x] D_ID %x not found on any channel- deferring", __func__, aep->at_rxid, did);
2592                         goto noresrc;
2593                 }
2594                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: [RX_ID 0x%x] D_ID 0x%06x found on Chan %d for S_ID 0x%06x", __func__, aep->at_rxid, did, chan, sid);
2595         } else {
2596                 chan = 0;
2597         }
2598
2599         /*
2600          * Find the PDB entry for this initiator
2601          */
2602         if (isp_find_pdb_by_sid(isp, chan, sid, &lp) == 0) {
2603                 /*
2604                  * If we're not in the port database terminate the exchange.
2605                  */
2606                 isp_prt(isp, ISP_LOGTINFO, "%s: [RX_ID 0x%x] D_ID 0x%06x found on Chan %d for S_ID 0x%06x wasn't in PDB already",
2607                     __func__, aep->at_rxid, did, chan, sid);
2608                 isp_endcmd(isp, aep, NIL_HANDLE, chan, ECMD_TERMINATE, 0);
2609                 return;
2610         }
2611         nphdl = lp->handle;
2612         wwn = lp->port_wwn;
2613
2614         /*
2615          * Get the tstate pointer
2616          */
2617         tptr = get_lun_statep(isp, chan, lun);
2618         if (tptr == NULL) {
2619                 tptr = get_lun_statep(isp, chan, CAM_LUN_WILDCARD);
2620                 if (tptr == NULL) {
2621                         isp_prt(isp, ISP_LOGWARN, "%s: [0x%x] no state pointer for lun %d or wildcard", __func__, aep->at_rxid, lun);
2622                         if (lun == 0) {
2623                                 isp_endcmd(isp, aep, nphdl, SCSI_STATUS_BUSY, 0);
2624                         } else {
2625                                 isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_CHECK_COND | ECMD_SVALID | (0x5 << 12) | (0x25 << 16), 0);
2626                         }
2627                         return;
2628                 }
2629         }
2630
2631         /*
2632          * Start any commands pending resources first.
2633          */
2634         if (tptr->restart_queue) {
2635                 inot_private_data_t *restart_queue = tptr->restart_queue;
2636                 tptr->restart_queue = NULL;
2637                 while (restart_queue) {
2638                         ntp = restart_queue;
2639                         restart_queue = ntp->rd.nt.nt_hba;
2640                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: restarting resrc deprived %x", __func__, ((at7_entry_t *)ntp->rd.data)->at_rxid);
2641                         isp_handle_platform_atio7(isp, (at7_entry_t *) ntp->rd.data);
2642                         isp_put_ntpd(isp, tptr, ntp);
2643                         /*
2644                          * If a recursion caused the restart queue to start to fill again,
2645                          * stop and splice the new list on top of the old list and restore
2646                          * it and go to noresrc.
2647                          */
2648                         if (tptr->restart_queue) {
2649                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: restart queue refilling", __func__);
2650                                 if (restart_queue) {
2651                                         ntp = tptr->restart_queue;
2652                                         tptr->restart_queue = restart_queue;
2653                                         while (restart_queue->rd.nt.nt_hba) {
2654                                                 restart_queue = restart_queue->rd.nt.nt_hba;
2655                                         }
2656                                         restart_queue->rd.nt.nt_hba = ntp;
2657                                 }
2658                                 goto noresrc;
2659                         }
2660                 }
2661         }
2662
2663         /*
2664          * If the f/w is out of resources, just send a BUSY status back.
2665          */
2666         if (aep->at_rxid == AT7_NORESRC_RXID) {
2667                 rls_lun_statep(isp, tptr);
2668                 isp_endcmd(isp, aep, nphdl, chan, SCSI_BUSY, 0);
2669                 return;
2670         }
2671
2672         /*
2673          * If we're out of resources, just send a BUSY status back.
2674          */
2675         atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
2676         if (atiop == NULL) {
2677                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atios", aep->at_rxid);
2678                 goto noresrc;
2679         }
2680
2681         oatp = isp_find_atpd(isp, tptr, aep->at_rxid);
2682         if (oatp) {
2683                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] tag wraparound in isp_handle_platforms_atio7 (N-Port Handle 0x%04x S_ID 0x%04x OX_ID 0x%04x) oatp state %d",
2684                     aep->at_rxid, nphdl, sid, aep->at_hdr.ox_id, oatp->state);
2685                 /*
2686                  * It's not a "no resource" condition- but we can treat it like one
2687                  */
2688                 goto noresrc;
2689         }
2690         atp = isp_get_atpd(isp, tptr, aep->at_rxid);
2691         if (atp == NULL) {
2692                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atps", aep->at_rxid);
2693                 goto noresrc;
2694         }
2695         atp->word3 = lp->prli_word3;
2696         atp->state = ATPD_STATE_ATIO;
2697         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
2698         tptr->atio_count--;
2699         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, atiop->ccb_h.path, "Take FREE ATIO count now %d\n", tptr->atio_count);
2700         atiop->init_id = nphdl;
2701         atiop->ccb_h.target_id = FCPARAM(isp, chan)->isp_loopid;
2702         atiop->ccb_h.target_lun = lun;
2703         atiop->sense_len = 0;
2704         cdbxlen = aep->at_cmnd.fcp_cmnd_alen_datadir >> FCP_CMND_ADDTL_CDBLEN_SHIFT;
2705         if (cdbxlen) {
2706                 isp_prt(isp, ISP_LOGWARN, "additional CDBLEN ignored");
2707         }
2708         cdbxlen = sizeof (aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb);
2709         ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb, cdbxlen);
2710         atiop->cdb_len = cdbxlen;
2711         atiop->ccb_h.status = CAM_CDB_RECVD;
2712         atiop->tag_id = atp->tag;
2713         switch (aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK) {
2714         case FCP_CMND_TASK_ATTR_SIMPLE:
2715                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
2716                 atiop->tag_action = MSG_SIMPLE_Q_TAG;
2717                 break;
2718         case FCP_CMND_TASK_ATTR_HEAD:
2719                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
2720                 atiop->tag_action = MSG_HEAD_OF_Q_TAG;
2721                 break;
2722         case FCP_CMND_TASK_ATTR_ORDERED:
2723                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
2724                 atiop->tag_action = MSG_ORDERED_Q_TAG;
2725                 break;
2726         default:
2727                 /* FALLTHROUGH */
2728         case FCP_CMND_TASK_ATTR_ACA:
2729         case FCP_CMND_TASK_ATTR_UNTAGGED:
2730                 atiop->tag_action = 0;
2731                 break;
2732         }
2733         atp->orig_datalen = aep->at_cmnd.cdb_dl.sf.fcp_cmnd_dl;
2734         atp->bytes_xfered = 0;
2735         atp->lun = lun;
2736         atp->nphdl = nphdl;
2737         atp->portid = sid;
2738         atp->oxid = aep->at_hdr.ox_id;
2739         atp->rxid = aep->at_hdr.rx_id;
2740         atp->cdb0 = atiop->cdb_io.cdb_bytes[0];
2741         atp->tattr = aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK;
2742         atp->state = ATPD_STATE_CAM;
2743         isp_prt(isp, ISP_LOGTDEBUG0, "ATIO7[0x%x] CDB=0x%x lun %d datalen %u", aep->at_rxid, atp->cdb0, lun, atp->orig_datalen);
2744         xpt_done((union ccb *)atiop);
2745         rls_lun_statep(isp, tptr);
2746         return;
2747 noresrc:
2748         if (atp) {
2749                 isp_put_atpd(isp, tptr, atp);
2750         }
2751         ntp = isp_get_ntpd(isp, tptr);
2752         if (ntp == NULL) {
2753                 rls_lun_statep(isp, tptr);
2754                 isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_BUSY, 0);
2755                 return;
2756         }
2757         memcpy(ntp->rd.data, aep, QENTRY_LEN);
2758         ntp->rd.nt.nt_hba = tptr->restart_queue;
2759         tptr->restart_queue = ntp;
2760         rls_lun_statep(isp, tptr);
2761 }
2762
2763
2764 /*
2765  * Handle starting an SRR (sequence retransmit request)
2766  * We get here when we've gotten the immediate notify
2767  * and the return of all outstanding CTIOs for this
2768  * transaction.
2769  */
2770 static void
2771 isp_handle_srr_start(ispsoftc_t *isp, tstate_t *tptr, atio_private_data_t *atp)
2772 {
2773         in_fcentry_24xx_t *inot;
2774         uint32_t srr_off, ccb_off, ccb_len, ccb_end;
2775         union ccb *ccb;
2776
2777         inot = (in_fcentry_24xx_t *)atp->srr;
2778         srr_off = inot->in_srr_reloff_lo | (inot->in_srr_reloff_hi << 16);
2779         ccb = atp->srr_ccb;
2780         atp->srr_ccb = NULL;
2781         atp->nsrr++;
2782         if (ccb == NULL) {
2783                 isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] null ccb", atp->tag);
2784                 goto fail;
2785         }
2786
2787         ccb_off = ccb->ccb_h.spriv_field0;
2788         ccb_len = ccb->csio.dxfer_len;
2789         ccb_end = (ccb_off == ~0)? ~0 : ccb_off + ccb_len;
2790
2791         switch (inot->in_srr_iu) {
2792         case R_CTL_INFO_SOLICITED_DATA:
2793                 /*
2794                  * We have to restart a FCP_DATA data out transaction
2795                  */
2796                 atp->sendst = 0;
2797                 atp->bytes_xfered = srr_off;
2798                 if (ccb_len == 0) {
2799                         isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x but current CCB doesn't transfer data", atp->tag, srr_off);
2800                         goto mdp;
2801                 }
2802                 if (srr_off < ccb_off || ccb_off > srr_off + ccb_len) {
2803                         isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x not covered by current CCB data range [0x%x..0x%x]", atp->tag, srr_off, ccb_off, ccb_end);
2804                         goto mdp;
2805                 }
2806                 isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x covered by current CCB data range [0x%x..0x%x]", atp->tag, srr_off, ccb_off, ccb_end);
2807                 break;
2808         case R_CTL_INFO_COMMAND_STATUS:
2809                 isp_prt(isp, ISP_LOGTINFO, "SRR[0x%x] Got an FCP RSP SRR- resending status", atp->tag);
2810                 atp->sendst = 1;
2811                 /*
2812                  * We have to restart a FCP_RSP IU transaction
2813                  */
2814                 break;
2815         case R_CTL_INFO_DATA_DESCRIPTOR:
2816                 /*
2817                  * We have to restart an FCP DATA in transaction
2818                  */
2819                 isp_prt(isp, ISP_LOGWARN, "Got an FCP DATA IN SRR- dropping");
2820                 goto fail;
2821
2822         default:
2823                 isp_prt(isp, ISP_LOGWARN, "Got an unknown information (%x) SRR- dropping", inot->in_srr_iu);
2824                 goto fail;
2825         }
2826
2827         /*
2828          * We can't do anything until this is acked, so we might as well start it now.
2829          * We aren't going to do the usual asynchronous ack issue because we need
2830          * to make sure this gets on the wire first.
2831          */
2832         if (isp_notify_ack(isp, inot)) {
2833                 isp_prt(isp, ISP_LOGWARN, "could not push positive ack for SRR- you lose");
2834                 goto fail;
2835         }
2836         isp_target_start_ctio(isp, ccb, FROM_SRR);
2837         return;
2838 fail:
2839         inot->in_reserved = 1;
2840         isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
2841         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2842         ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
2843         isp_complete_ctio(ccb);
2844         return;
2845 mdp:
2846         if (isp_notify_ack(isp, inot)) {
2847                 isp_prt(isp, ISP_LOGWARN, "could not push positive ack for SRR- you lose");
2848                 goto fail;
2849         }
2850         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2851         ccb->ccb_h.status = CAM_MESSAGE_RECV;
2852         /*
2853          * This is not a strict interpretation of MDP, but it's close
2854          */
2855         ccb->csio.msg_ptr = &ccb->csio.sense_data.sense_buf[SSD_FULL_SIZE - 16];
2856         ccb->csio.msg_len = 7;
2857         ccb->csio.msg_ptr[0] = MSG_EXTENDED;
2858         ccb->csio.msg_ptr[1] = 5;
2859         ccb->csio.msg_ptr[2] = 0;       /* modify data pointer */
2860         ccb->csio.msg_ptr[3] = srr_off >> 24;
2861         ccb->csio.msg_ptr[4] = srr_off >> 16;
2862         ccb->csio.msg_ptr[5] = srr_off >> 8;
2863         ccb->csio.msg_ptr[6] = srr_off;
2864         isp_complete_ctio(ccb);
2865 }
2866
2867
2868 static void
2869 isp_handle_srr_notify(ispsoftc_t *isp, void *inot_raw)
2870 {
2871         tstate_t *tptr;
2872         in_fcentry_24xx_t *inot = inot_raw;
2873         atio_private_data_t *atp;
2874         uint32_t tag = inot->in_rxid;
2875         uint32_t bus = inot->in_vpidx;
2876
2877         if (!IS_24XX(isp)) {
2878                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot_raw);
2879                 return;
2880         }
2881
2882         tptr = get_lun_statep_from_tag(isp, bus, tag);
2883         if (tptr == NULL) {
2884                 isp_prt(isp, ISP_LOGERR, "%s: cannot find tptr for tag %x in SRR Notify", __func__, tag);
2885                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
2886                 return;
2887         }
2888         atp = isp_find_atpd(isp, tptr, tag);
2889         if (atp == NULL) {
2890                 rls_lun_statep(isp, tptr);
2891                 isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x in SRR Notify", __func__, tag);
2892                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
2893                 return;
2894         }
2895         atp->srr_notify_rcvd = 1;
2896         memcpy(atp->srr, inot, sizeof (atp->srr));
2897         isp_prt(isp, ISP_LOGTINFO /* ISP_LOGTDEBUG0 */, "SRR[0x%x] inot->in_rxid flags 0x%x srr_iu=%x reloff 0x%x", inot->in_rxid, inot->in_flags, inot->in_srr_iu,
2898             inot->in_srr_reloff_lo | (inot->in_srr_reloff_hi << 16));
2899         if (atp->srr_ccb)
2900                 isp_handle_srr_start(isp, tptr, atp);
2901         rls_lun_statep(isp, tptr);
2902 }
2903
2904 static void
2905 isp_handle_platform_ctio(ispsoftc_t *isp, void *arg)
2906 {
2907         union ccb *ccb;
2908         int sentstatus = 0, ok = 0, notify_cam = 0, resid = 0, failure = 0;
2909         tstate_t *tptr = NULL;
2910         atio_private_data_t *atp = NULL;
2911         int bus;
2912         uint32_t handle, moved_data = 0, data_requested;
2913
2914         /*
2915          * CTIO handles are 16 bits.
2916          * CTIO2 and CTIO7 are 32 bits.
2917          */
2918
2919         if (IS_SCSI(isp)) {
2920                 handle = ((ct_entry_t *)arg)->ct_syshandle;
2921         } else {
2922                 handle = ((ct2_entry_t *)arg)->ct_syshandle;
2923         }
2924         ccb = isp_find_xs_tgt(isp, handle);
2925         if (ccb == NULL) {
2926                 isp_print_bytes(isp, "null ccb in isp_handle_platform_ctio", QENTRY_LEN, arg);
2927                 return;
2928         }
2929         isp_destroy_tgt_handle(isp, handle);
2930         data_requested = PISP_PCMD(ccb)->datalen;
2931         isp_free_pcmd(isp, ccb);
2932         if (isp->isp_nactive) {
2933                 isp->isp_nactive--;
2934         }
2935
2936         bus = XS_CHANNEL(ccb);
2937         tptr = get_lun_statep(isp, bus, XS_LUN(ccb));
2938         if (tptr == NULL) {
2939                 tptr = get_lun_statep(isp, bus, CAM_LUN_WILDCARD);
2940         }
2941         if (tptr == NULL) {
2942                 isp_prt(isp, ISP_LOGERR, "%s: cannot find tptr for tag %x after I/O", __func__, ccb->csio.tag_id);
2943                 return;
2944         }
2945
2946         if (IS_24XX(isp)) {
2947                 atp = isp_find_atpd(isp, tptr, ((ct7_entry_t *)arg)->ct_rxid);
2948         } else if (IS_FC(isp)) {
2949                 atp = isp_find_atpd(isp, tptr, ((ct2_entry_t *)arg)->ct_rxid);
2950         } else {
2951                 atp = isp_find_atpd(isp, tptr, ((ct_entry_t *)arg)->ct_fwhandle);
2952         }
2953         if (atp == NULL) {
2954                 /*
2955                  * In case of target mode disable at least ISP2532 return
2956                  * invalid zero ct_rxid value.  Try to workaround that using
2957                  * tag_id from the CCB, pointed by valid ct_syshandle.
2958                  */
2959                 atp = isp_find_atpd(isp, tptr, ccb->csio.tag_id);
2960         }
2961         if (atp == NULL) {
2962                 rls_lun_statep(isp, tptr);
2963                 isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x after I/O", __func__, ccb->csio.tag_id);
2964                 return;
2965         }
2966         KASSERT((atp->ctcnt > 0), ("ctio count not greater than zero"));
2967         atp->bytes_in_transit -= data_requested;
2968         atp->ctcnt -= 1;
2969         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2970
2971         if (IS_24XX(isp)) {
2972                 ct7_entry_t *ct = arg;
2973
2974                 if (ct->ct_nphdl == CT7_SRR) {
2975                         atp->srr_ccb = ccb;
2976                         if (atp->srr_notify_rcvd)
2977                                 isp_handle_srr_start(isp, tptr, atp);
2978                         rls_lun_statep(isp, tptr);
2979                         return;
2980                 }
2981                 if (ct->ct_nphdl == CT_HBA_RESET) {
2982                         failure = CAM_UNREC_HBA_ERROR;
2983                 } else {
2984                         sentstatus = ct->ct_flags & CT7_SENDSTATUS;
2985                         ok = (ct->ct_nphdl == CT7_OK);
2986                         notify_cam = (ct->ct_header.rqs_seqno & ATPD_SEQ_NOTIFY_CAM) != 0;
2987                         if ((ct->ct_flags & CT7_DATAMASK) != CT7_NO_DATA) {
2988                                 resid = ct->ct_resid;
2989                                 moved_data = data_requested - resid;
2990                         }
2991                 }
2992                 isp_prt(isp, ok? ISP_LOGTDEBUG0 : ISP_LOGWARN, "%s: CTIO7[%x] seq %u nc %d sts 0x%x flg 0x%x sns %d resid %d %s", __func__, ct->ct_rxid, ATPD_GET_SEQNO(ct),
2993                    notify_cam, ct->ct_nphdl, ct->ct_flags, (ccb->ccb_h.status & CAM_SENT_SENSE) != 0, resid, sentstatus? "FIN" : "MID");
2994         } else if (IS_FC(isp)) {
2995                 ct2_entry_t *ct = arg;
2996                 if (ct->ct_status == CT_SRR) {
2997                         atp->srr_ccb = ccb;
2998                         if (atp->srr_notify_rcvd)
2999                                 isp_handle_srr_start(isp, tptr, atp);
3000                         rls_lun_statep(isp, tptr);
3001                         isp_target_putback_atio(ccb);
3002                         return;
3003                 }
3004                 if (ct->ct_status == CT_HBA_RESET) {
3005                         failure = CAM_UNREC_HBA_ERROR;
3006                 } else {
3007                         sentstatus = ct->ct_flags & CT2_SENDSTATUS;
3008                         ok = (ct->ct_status & ~QLTM_SVALID) == CT_OK;
3009                         notify_cam = (ct->ct_header.rqs_seqno & ATPD_SEQ_NOTIFY_CAM) != 0;
3010                         if ((ct->ct_flags & CT2_DATAMASK) != CT2_NO_DATA) {
3011                                 resid = ct->ct_resid;
3012                                 moved_data = data_requested - resid;
3013                         }
3014                 }
3015                 isp_prt(isp, ok? ISP_LOGTDEBUG0 : ISP_LOGWARN, "%s: CTIO2[%x] seq %u nc %d sts 0x%x flg 0x%x sns %d resid %d %s", __func__, ct->ct_rxid, ATPD_GET_SEQNO(ct),
3016                     notify_cam, ct->ct_status, ct->ct_flags, (ccb->ccb_h.status & CAM_SENT_SENSE) != 0, resid, sentstatus? "FIN" : "MID");
3017         } else {
3018                 ct_entry_t *ct = arg;
3019
3020                 if (ct->ct_status == (CT_HBA_RESET & 0xff)) {
3021                         failure = CAM_UNREC_HBA_ERROR;
3022                 } else {
3023                         sentstatus = ct->ct_flags & CT_SENDSTATUS;
3024                         ok = (ct->ct_status  & ~QLTM_SVALID) == CT_OK;
3025                         notify_cam = (ct->ct_header.rqs_seqno & ATPD_SEQ_NOTIFY_CAM) != 0;
3026                 }
3027                 if ((ct->ct_flags & CT_DATAMASK) != CT_NO_DATA) {
3028                         resid = ct->ct_resid;
3029                         moved_data = data_requested - resid;
3030                 }
3031                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO[%x] seq %u nc %d tag %x S_ID 0x%x lun %d sts %x flg %x resid %d %s", __func__, ct->ct_fwhandle, ATPD_GET_SEQNO(ct),
3032                     notify_cam, ct->ct_tag_val, ct->ct_iid, ct->ct_lun, ct->ct_status, ct->ct_flags, resid, sentstatus? "FIN" : "MID");
3033         }
3034         if (ok) {
3035                 if (moved_data) {
3036                         atp->bytes_xfered += moved_data;
3037                         ccb->csio.resid = atp->orig_datalen - atp->bytes_xfered - atp->bytes_in_transit;
3038                 }
3039                 if (sentstatus && (ccb->ccb_h.flags & CAM_SEND_SENSE)) {
3040                         ccb->ccb_h.status |= CAM_SENT_SENSE;
3041                 }
3042                 ccb->ccb_h.status |= CAM_REQ_CMP;
3043         } else {
3044                 notify_cam = 1;
3045                 if (failure == CAM_UNREC_HBA_ERROR)
3046                         ccb->ccb_h.status |= CAM_UNREC_HBA_ERROR;
3047                 else
3048                         ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
3049         }
3050         atp->state = ATPD_STATE_PDON;
3051         rls_lun_statep(isp, tptr);
3052
3053         /*
3054          * We never *not* notify CAM when there has been any error (ok == 0),
3055          * so we never need to do an ATIO putback if we're not notifying CAM.
3056          */
3057         isp_prt(isp, ISP_LOGTDEBUG0, "%s CTIO[0x%x] done (ok=%d nc=%d nowsendstatus=%d ccb ss=%d)",
3058             (sentstatus)? "  FINAL " : "MIDTERM ", atp->tag, ok, notify_cam, atp->sendst, (ccb->ccb_h.flags & CAM_SEND_STATUS) != 0);
3059         if (notify_cam == 0) {
3060                 if (atp->sendst) {
3061                         isp_target_start_ctio(isp, ccb, FROM_CTIO_DONE);
3062                 }
3063                 return;
3064         }
3065
3066         /*
3067          * We're telling CAM we're done with this CTIO transaction.
3068          *
3069          * 24XX cards never need an ATIO put back.
3070          *
3071          * Other cards need one put back only on error.
3072          * In the latter case, a timeout will re-fire
3073          * and try again in case we didn't have
3074          * queue resources to do so at first. In any case,
3075          * once the putback is done we do the completion
3076          * call.
3077          */
3078         if (ok || IS_24XX(isp)) {
3079                 isp_complete_ctio(ccb);
3080         } else {
3081                 isp_target_putback_atio(ccb);
3082         }
3083 }
3084
3085 static void
3086 isp_handle_platform_notify_scsi(ispsoftc_t *isp, in_entry_t *inot)
3087 {
3088         isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
3089 }
3090
3091 static void
3092 isp_handle_platform_notify_fc(ispsoftc_t *isp, in_fcentry_t *inp)
3093 {
3094         int needack = 1;
3095         switch (inp->in_status) {
3096         case IN_PORT_LOGOUT:
3097                 /*
3098                  * XXX: Need to delete this initiator's WWN from the database
3099                  * XXX: Need to send this LOGOUT upstream
3100                  */
3101                 isp_prt(isp, ISP_LOGWARN, "port logout of S_ID 0x%x", inp->in_iid);
3102                 break;
3103         case IN_PORT_CHANGED:
3104                 isp_prt(isp, ISP_LOGWARN, "port changed for S_ID 0x%x", inp->in_iid);
3105                 break;
3106         case IN_GLOBAL_LOGO:
3107                 isp_del_all_wwn_entries(isp, 0);
3108                 isp_prt(isp, ISP_LOGINFO, "all ports logged out");
3109                 break;
3110         case IN_ABORT_TASK:
3111         {
3112                 tstate_t *tptr;
3113                 uint16_t lun;
3114                 uint32_t loopid;
3115                 uint64_t wwn;
3116                 atio_private_data_t *atp;
3117                 fcportdb_t *lp;
3118                 struct ccb_immediate_notify *inot = NULL;
3119
3120                 if (ISP_CAP_SCCFW(isp)) {
3121                         lun = inp->in_scclun;
3122                 } else {
3123                         lun = inp->in_lun;
3124                 }
3125                 if (ISP_CAP_2KLOGIN(isp)) {
3126                         loopid = ((in_fcentry_e_t *)inp)->in_iid;
3127                 } else {
3128                         loopid = inp->in_iid;
3129                 }
3130                 if (isp_find_pdb_by_loopid(isp, 0, loopid, &lp)) {
3131                         wwn = lp->port_wwn;
3132                 } else {
3133                         wwn = INI_ANY;
3134                 }
3135                 tptr = get_lun_statep(isp, 0, lun);
3136                 if (tptr == NULL) {
3137                         tptr = get_lun_statep(isp, 0, CAM_LUN_WILDCARD);
3138                         if (tptr == NULL) {
3139                                 isp_prt(isp, ISP_LOGWARN, "ABORT TASK for lun %u- but no tstate", lun);
3140                                 return;
3141                         }
3142                 }
3143                 atp = isp_find_atpd(isp, tptr, inp->in_seqid);
3144
3145                 if (atp) {
3146                         inot = (struct ccb_immediate_notify *) SLIST_FIRST(&tptr->inots);
3147                         isp_prt(isp, ISP_LOGTDEBUG0, "ABORT TASK RX_ID %x WWN 0x%016llx state %d", inp->in_seqid, (unsigned long long) wwn, atp->state);
3148                         if (inot) {
3149                                 tptr->inot_count--;
3150                                 SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle);
3151                                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, inot->ccb_h.path, "%s: Take FREE INOT count now %d\n", __func__, tptr->inot_count);
3152                         } else {
3153                                 ISP_PATH_PRT(isp, ISP_LOGWARN, tptr->owner, "out of INOT structures\n");
3154                         }
3155                 } else {
3156                         ISP_PATH_PRT(isp, ISP_LOGWARN, tptr->owner, "abort task RX_ID %x from wwn 0x%016llx, state unknown\n", inp->in_seqid, wwn);
3157                 }
3158                 if (inot) {
3159                         isp_notify_t tmp, *nt = &tmp;
3160                         ISP_MEMZERO(nt, sizeof (isp_notify_t));
3161                         nt->nt_hba = isp;
3162                         nt->nt_tgt = FCPARAM(isp, 0)->isp_wwpn;
3163                         nt->nt_wwn = wwn;
3164                         nt->nt_nphdl = loopid;
3165                         nt->nt_sid = PORT_ANY;
3166                         nt->nt_did = PORT_ANY;
3167                         nt->nt_lun = lun;
3168                         nt->nt_need_ack = 1;
3169                         nt->nt_channel = 0;
3170                         nt->nt_ncode = NT_ABORT_TASK;
3171                         nt->nt_lreserved = inot;
3172                         isp_handle_platform_target_tmf(isp, nt);
3173                         needack = 0;
3174                 }
3175                 rls_lun_statep(isp, tptr);
3176                 break;
3177         }
3178         default:
3179                 break;
3180         }
3181         if (needack) {
3182                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inp);
3183         }
3184 }
3185
3186 static void
3187 isp_handle_platform_notify_24xx(ispsoftc_t *isp, in_fcentry_24xx_t *inot)
3188 {
3189         uint16_t nphdl;
3190         uint16_t prli_options = 0;
3191         uint32_t portid;
3192         fcportdb_t *lp;
3193         uint8_t *ptr = NULL;
3194         uint64_t wwn;
3195
3196         nphdl = inot->in_nphdl;
3197         if (nphdl != NIL_HANDLE) {
3198                 portid = inot->in_portid_hi << 16 | inot->in_portid_lo;
3199         } else {
3200                 portid = PORT_ANY;
3201         }
3202
3203         switch (inot->in_status) {
3204         case IN24XX_ELS_RCVD:
3205         {
3206                 char buf[16], *msg;
3207                 int chan = ISP_GET_VPIDX(isp, inot->in_vpidx);
3208
3209                 /*
3210                  * Note that we're just getting notification that an ELS was received
3211                  * (possibly with some associated information sent upstream). This is
3212                  * *not* the same as being given the ELS frame to accept or reject.
3213                  */
3214                 switch (inot->in_status_subcode) {
3215                 case LOGO:
3216                         msg = "LOGO";
3217                         if (ISP_FW_NEWER_THAN(isp, 4, 0, 25)) {
3218                                 ptr = (uint8_t *)inot;  /* point to unswizzled entry! */
3219                                 wwn =   (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF])   << 56) |
3220                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+1]) << 48) |
3221                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+2]) << 40) |
3222                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+3]) << 32) |
3223                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+4]) << 24) |
3224                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+5]) << 16) |
3225                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+6]) <<  8) |
3226                                         (((uint64_t) ptr[IN24XX_LOGO_WWPN_OFF+7]));
3227                         } else {
3228                                 wwn = INI_ANY;
3229                         }
3230                         isp_del_wwn_entry(isp, chan, wwn, nphdl, portid);
3231                         break;
3232                 case PRLO:
3233                         msg = "PRLO";
3234                         break;
3235                 case PLOGI:
3236                 case PRLI:
3237                         /*
3238                          * Treat PRLI the same as PLOGI and make a database entry for it.
3239                          */
3240                         if (inot->in_status_subcode == PLOGI) {
3241                                 msg = "PLOGI";
3242                         } else {
3243                                 prli_options = inot->in_prli_options;
3244                                 msg = "PRLI";
3245                         }
3246                         if (ISP_FW_NEWER_THAN(isp, 4, 0, 25)) {
3247                                 ptr = (uint8_t *)inot;  /* point to unswizzled entry! */
3248                                 wwn =   (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF])   << 56) |
3249                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+1]) << 48) |
3250                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+2]) << 40) |
3251                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+3]) << 32) |
3252                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+4]) << 24) |
3253                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+5]) << 16) |
3254                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+6]) <<  8) |
3255                                         (((uint64_t) ptr[IN24XX_PLOGI_WWPN_OFF+7]));
3256                         } else {
3257                                 wwn = INI_NONE;
3258                         }
3259                         isp_add_wwn_entry(isp, chan, wwn, nphdl, portid, prli_options);
3260                         break;
3261                 case PDISC:
3262                         msg = "PDISC";
3263                         break;
3264                 case ADISC:
3265                         msg = "ADISC";
3266                         break;
3267                 default:
3268                         ISP_SNPRINTF(buf, sizeof (buf), "ELS 0x%x", inot->in_status_subcode);
3269                         msg = buf;
3270                         break;
3271                 }
3272                 if (inot->in_flags & IN24XX_FLAG_PUREX_IOCB) {
3273                         isp_prt(isp, ISP_LOGERR, "%s Chan %d ELS N-port handle %x PortID 0x%06x marked as needing a PUREX response", msg, chan, nphdl, portid);
3274                         break;
3275                 }
3276                 isp_prt(isp, ISP_LOGTDEBUG0, "%s Chan %d ELS N-port handle %x PortID 0x%06x RX_ID 0x%x OX_ID 0x%x", msg, chan, nphdl, portid,
3277                     inot->in_rxid, inot->in_oxid);
3278                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
3279                 break;
3280         }
3281
3282         case IN24XX_PORT_LOGOUT:
3283                 ptr = "PORT LOGOUT";
3284                 if (isp_find_pdb_by_loopid(isp, ISP_GET_VPIDX(isp, inot->in_vpidx), nphdl, &lp)) {
3285                         isp_del_wwn_entry(isp, ISP_GET_VPIDX(isp, inot->in_vpidx), lp->port_wwn, nphdl, lp->portid);
3286                 }
3287                 /* FALLTHROUGH */
3288         case IN24XX_PORT_CHANGED:
3289                 if (ptr == NULL) {
3290                         ptr = "PORT CHANGED";
3291                 }
3292                 /* FALLTHROUGH */
3293         case IN24XX_LIP_RESET:
3294                 if (ptr == NULL) {
3295                         ptr = "LIP RESET";
3296                 }
3297                 isp_prt(isp, ISP_LOGINFO, "Chan %d %s (sub-status 0x%x) for N-port handle 0x%x", ISP_GET_VPIDX(isp, inot->in_vpidx), ptr, inot->in_status_subcode, nphdl);
3298
3299                 /*
3300                  * All subcodes here are irrelevant. What is relevant
3301                  * is that we need to terminate all active commands from
3302                  * this initiator (known by N-port handle).
3303                  */
3304                 /* XXX IMPLEMENT XXX */
3305                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
3306                 break;
3307
3308         case IN24XX_SRR_RCVD:
3309 #ifdef  ISP_TARGET_MODE
3310                 isp_handle_srr_notify(isp, inot);
3311                 break;
3312 #else
3313                 if (ptr == NULL) {
3314                         ptr = "SRR RCVD";
3315                 }
3316                 /* FALLTHROUGH */
3317 #endif
3318         case IN24XX_LINK_RESET:
3319                 if (ptr == NULL) {
3320                         ptr = "LINK RESET";
3321                 }
3322         case IN24XX_LINK_FAILED:
3323                 if (ptr == NULL) {
3324                         ptr = "LINK FAILED";
3325                 }
3326         default:
3327                 isp_prt(isp, ISP_LOGWARN, "Chan %d %s", ISP_GET_VPIDX(isp, inot->in_vpidx), ptr);
3328                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
3329                 break;
3330         }
3331 }
3332
3333 static int
3334 isp_handle_platform_target_notify_ack(ispsoftc_t *isp, isp_notify_t *mp)
3335 {
3336
3337         if (isp->isp_state != ISP_RUNSTATE) {
3338                 isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) acked- h/w not ready (dropping)", mp->nt_ncode, mp->nt_lreserved != NULL);
3339                 return (0);
3340         }
3341
3342         /*
3343          * This case is for a Task Management Function, which shows up as an ATIO7 entry.
3344          */
3345         if (IS_24XX(isp) && mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ATIO) {
3346                 ct7_entry_t local, *cto = &local;
3347                 at7_entry_t *aep = (at7_entry_t *)mp->nt_lreserved;
3348                 fcportdb_t *lp;
3349                 uint32_t sid;
3350                 uint16_t nphdl;
3351
3352                 sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2];
3353                 if (isp_find_pdb_by_sid(isp, mp->nt_channel, sid, &lp)) {
3354                         nphdl = lp->handle;
3355                 } else {
3356                         nphdl = NIL_HANDLE;
3357                 }
3358                 ISP_MEMZERO(&local, sizeof (local));
3359                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
3360                 cto->ct_header.rqs_entry_count = 1;
3361                 cto->ct_nphdl = nphdl;
3362                 cto->ct_rxid = aep->at_rxid;
3363                 cto->ct_vpidx = mp->nt_channel;
3364                 cto->ct_iid_lo = sid;
3365                 cto->ct_iid_hi = sid >> 16;
3366                 cto->ct_oxid = aep->at_hdr.ox_id;
3367                 cto->ct_flags = CT7_SENDSTATUS|CT7_NOACK|CT7_NO_DATA|CT7_FLAG_MODE1;
3368                 cto->ct_flags |= (aep->at_ta_len >> 12) << CT7_TASK_ATTR_SHIFT;
3369                 return (isp_target_put_entry(isp, &local));
3370         }
3371
3372         /*
3373          * This case is for a responding to an ABTS frame
3374          */
3375         if (IS_24XX(isp) && mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) {
3376
3377                 /*
3378                  * Overload nt_need_ack here to mark whether we've terminated the associated command.
3379                  */
3380                 if (mp->nt_need_ack) {
3381                         uint8_t storage[QENTRY_LEN];
3382                         ct7_entry_t *cto = (ct7_entry_t *) storage;
3383                         abts_t *abts = (abts_t *)mp->nt_lreserved;
3384
3385                         ISP_MEMZERO(cto, sizeof (ct7_entry_t));
3386                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: [%x] terminating after ABTS received", __func__, abts->abts_rxid_task);
3387                         cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
3388                         cto->ct_header.rqs_entry_count = 1;
3389                         cto->ct_nphdl = mp->nt_nphdl;
3390                         cto->ct_rxid = abts->abts_rxid_task;
3391                         cto->ct_iid_lo = mp->nt_sid;
3392                         cto->ct_iid_hi = mp->nt_sid >> 16;
3393                         cto->ct_oxid = abts->abts_ox_id;
3394                         cto->ct_vpidx = mp->nt_channel;
3395                         cto->ct_flags = CT7_NOACK|CT7_TERMINATE;
3396                         if (isp_target_put_entry(isp, cto)) {
3397                                 return (ENOMEM);
3398                         }
3399                         mp->nt_need_ack = 0;
3400                 }
3401                 if (isp_acknak_abts(isp, mp->nt_lreserved, 0) == ENOMEM) {
3402                         return (ENOMEM);
3403                 } else {
3404                         return (0);
3405                 }
3406         }
3407
3408         /*
3409          * Handle logout cases here
3410          */
3411         if (mp->nt_ncode == NT_GLOBAL_LOGOUT) {
3412                 isp_del_all_wwn_entries(isp, mp->nt_channel);
3413         }
3414
3415         if (mp->nt_ncode == NT_LOGOUT) {
3416                 if (!IS_2100(isp) && IS_FC(isp)) {
3417                         isp_del_wwn_entries(isp, mp);
3418                 }
3419         }
3420
3421         /*
3422          * General purpose acknowledgement
3423          */
3424         if (mp->nt_need_ack) {
3425                 isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) being acked", mp->nt_ncode, mp->nt_lreserved != NULL);
3426                 /*
3427                  * Don't need to use the guaranteed send because the caller can retry
3428                  */
3429                 return (isp_notify_ack(isp, mp->nt_lreserved));
3430         }
3431         return (0);
3432 }
3433
3434 /*
3435  * Handle task management functions.
3436  *
3437  * We show up here with a notify structure filled out.
3438  *
3439  * The nt_lreserved tag points to the original queue entry
3440  */
3441 static void
3442 isp_handle_platform_target_tmf(ispsoftc_t *isp, isp_notify_t *notify)
3443 {
3444         tstate_t *tptr;
3445         fcportdb_t *lp;
3446         struct ccb_immediate_notify *inot;
3447         inot_private_data_t *ntp = NULL;
3448         lun_id_t lun;
3449
3450         isp_prt(isp, ISP_LOGTDEBUG0, "%s: code 0x%x sid  0x%x tagval 0x%016llx chan %d lun 0x%x", __func__, notify->nt_ncode,
3451             notify->nt_sid, (unsigned long long) notify->nt_tagval, notify->nt_channel, notify->nt_lun);
3452         /*
3453          * NB: This assignment is necessary because of tricky type conversion.
3454          * XXX: This is tricky and I need to check this. If the lun isn't known
3455          * XXX: for the task management function, it does not of necessity follow
3456          * XXX: that it should go up stream to the wildcard listener.
3457          */
3458         if (notify->nt_lun == LUN_ANY) {
3459                 lun = CAM_LUN_WILDCARD;
3460         } else {
3461                 lun = notify->nt_lun;
3462         }
3463         tptr = get_lun_statep(isp, notify->nt_channel, lun);
3464         if (tptr == NULL) {
3465                 tptr = get_lun_statep(isp, notify->nt_channel, CAM_LUN_WILDCARD);
3466                 if (tptr == NULL) {
3467                         isp_prt(isp, ISP_LOGWARN, "%s: no state pointer found for chan %d lun 0x%x", __func__, notify->nt_channel, lun);
3468                         goto bad;
3469                 }
3470         }
3471         inot = (struct ccb_immediate_notify *) SLIST_FIRST(&tptr->inots);
3472         if (inot == NULL) {
3473                 isp_prt(isp, ISP_LOGWARN, "%s: out of immediate notify structures for chan %d lun 0x%x", __func__, notify->nt_channel, lun);
3474                 goto bad;
3475         }
3476
3477         if (isp_find_pdb_by_sid(isp, notify->nt_channel, notify->nt_sid, &lp) == 0) {
3478                 inot->initiator_id = CAM_TARGET_WILDCARD;
3479         } else {
3480                 inot->initiator_id = lp->handle;
3481         }
3482         inot->seq_id = notify->nt_tagval;
3483         inot->tag_id = notify->nt_tagval >> 32;
3484
3485         switch (notify->nt_ncode) {
3486         case NT_ABORT_TASK:
3487                 isp_target_mark_aborted_early(isp, tptr, inot->tag_id);
3488                 inot->arg = MSG_ABORT_TASK;
3489                 break;
3490         case NT_ABORT_TASK_SET:
3491                 isp_target_mark_aborted_early(isp, tptr, TAG_ANY);
3492                 inot->arg = MSG_ABORT_TASK_SET;
3493                 break;
3494         case NT_CLEAR_ACA:
3495                 inot->arg = MSG_CLEAR_ACA;
3496                 break;
3497         case NT_CLEAR_TASK_SET:
3498                 inot->arg = MSG_CLEAR_TASK_SET;
3499                 break;
3500         case NT_LUN_RESET:
3501                 inot->arg = MSG_LOGICAL_UNIT_RESET;
3502                 break;
3503         case NT_TARGET_RESET:
3504                 inot->arg = MSG_TARGET_RESET;
3505                 break;
3506         default:
3507                 isp_prt(isp, ISP_LOGWARN, "%s: unknown TMF code 0x%x for chan %d lun 0x%x", __func__, notify->nt_ncode, notify->nt_channel, lun);
3508                 goto bad;
3509         }
3510
3511         ntp = isp_get_ntpd(isp, tptr);
3512         if (ntp == NULL) {
3513                 isp_prt(isp, ISP_LOGWARN, "%s: out of inotify private structures", __func__);
3514                 goto bad;
3515         }
3516         ISP_MEMCPY(&ntp->rd.nt, notify, sizeof (isp_notify_t));
3517         if (notify->nt_lreserved) {
3518                 ISP_MEMCPY(&ntp->rd.data, notify->nt_lreserved, QENTRY_LEN);
3519                 ntp->rd.nt.nt_lreserved = &ntp->rd.data;
3520         }
3521         ntp->rd.seq_id = notify->nt_tagval;
3522         ntp->rd.tag_id = notify->nt_tagval >> 32;
3523
3524         tptr->inot_count--;
3525         SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle);
3526         rls_lun_statep(isp, tptr);
3527         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, inot->ccb_h.path, "%s: Take FREE INOT count now %d\n", __func__, tptr->inot_count);
3528         inot->ccb_h.status = CAM_MESSAGE_RECV;
3529         xpt_done((union ccb *)inot);
3530         return;
3531 bad:
3532         if (tptr) {
3533                 rls_lun_statep(isp, tptr);
3534         }
3535         if (notify->nt_need_ack && notify->nt_lreserved) {
3536                 if (((isphdr_t *)notify->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) {
3537                         if (isp_acknak_abts(isp, notify->nt_lreserved, ENOMEM)) {
3538                                 isp_prt(isp, ISP_LOGWARN, "you lose- unable to send an ACKNAK");
3539                         }
3540                 } else {
3541                         isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, notify->nt_lreserved);
3542                 }
3543         }
3544 }
3545
3546 /*
3547  * Find the associated private data and mark it as dead so
3548  * we don't try to work on it any further.
3549  */
3550 static void
3551 isp_target_mark_aborted(ispsoftc_t *isp, union ccb *ccb)
3552 {
3553         tstate_t *tptr;
3554         atio_private_data_t *atp;
3555         union ccb *accb = ccb->cab.abort_ccb;
3556
3557         tptr = get_lun_statep(isp, XS_CHANNEL(accb), XS_LUN(accb));
3558         if (tptr == NULL) {
3559                 tptr = get_lun_statep(isp, XS_CHANNEL(accb), CAM_LUN_WILDCARD);
3560                 if (tptr == NULL) {
3561                         ccb->ccb_h.status = CAM_REQ_INVALID;
3562                         return;
3563                 }
3564         }
3565
3566         atp = isp_find_atpd(isp, tptr, accb->atio.tag_id);
3567         if (atp == NULL) {
3568                 ccb->ccb_h.status = CAM_REQ_INVALID;
3569         } else {
3570                 atp->dead = 1;
3571                 ccb->ccb_h.status = CAM_REQ_CMP;
3572         }
3573         rls_lun_statep(isp, tptr);
3574 }
3575
3576 static void
3577 isp_target_mark_aborted_early(ispsoftc_t *isp, tstate_t *tptr, uint32_t tag_id)
3578 {
3579         atio_private_data_t *atp;
3580         inot_private_data_t *restart_queue = tptr->restart_queue;
3581
3582         /*
3583          * First, clean any commands pending restart
3584          */
3585         tptr->restart_queue = NULL;
3586         while (restart_queue) {
3587                 uint32_t this_tag_id;
3588                 inot_private_data_t *ntp = restart_queue;
3589
3590                 restart_queue = ntp->rd.nt.nt_hba;
3591
3592                 if (IS_24XX(isp)) {
3593                         this_tag_id = ((at7_entry_t *)ntp->rd.data)->at_rxid;
3594                 } else {
3595                         this_tag_id = ((at2_entry_t *)ntp->rd.data)->at_rxid;
3596                 }
3597                 if ((uint64_t)tag_id == TAG_ANY || tag_id == this_tag_id) {
3598                         isp_put_ntpd(isp, tptr, ntp);
3599                 } else {
3600                         ntp->rd.nt.nt_hba = tptr->restart_queue;
3601                         tptr->restart_queue = ntp;
3602                 }
3603         }
3604
3605         /*
3606          * Now mark other ones dead as well.
3607          */
3608         for (atp = tptr->atpool; atp < &tptr->atpool[ATPDPSIZE]; atp++) {
3609                 if ((uint64_t)tag_id == TAG_ANY || atp->tag == tag_id) {
3610                         atp->dead = 1;
3611                 }
3612         }
3613 }
3614
3615
3616 #ifdef  ISP_INTERNAL_TARGET
3617 //#define       ISP_SEPARATE_STATUS     1
3618 #define ISP_MULTI_CCBS          1
3619 #if defined(ISP_MULTI_CCBS) && !defined(ISP_SEPARATE_STATUS)
3620 #define ISP_SEPARATE_STATUS 1
3621 #endif
3622
3623 typedef struct periph_private_data_t {
3624         union ccb *ccb;                 /* original ATIO or Immediate Notify */
3625         unsigned long   offset;         /* current offset */
3626         int             sequence;       /* current CTIO sequence */
3627         int             ctio_cnt;       /* current # of ctio's outstanding */
3628         int
3629                 status_sent     : 1,
3630                 on_queue        : 1;    /* on restart queue */
3631 } ppd_t;
3632 /*
3633  * Each ATIO we allocate will have periph private data associated with it
3634  * that maintains per-command state. This private to each ATIO.
3635  */
3636 #define ATIO_PPD(ccb)           ((ppd_t *)(((struct ccb_hdr *)ccb)->ppriv_ptr0))
3637 /*
3638  * Each CTIO we send downstream will get a pointer to the ATIO itself
3639  * so that on completion we can retrieve that pointer.
3640  */
3641 #define ccb_atio                ppriv_ptr1
3642 #define ccb_inot                ppriv_ptr1
3643
3644 /*
3645  * Each CTIO we send downstream will contain a sequence number
3646  */
3647 #define CTIO_SEQ(ccb)           ccb->ccb_h.ppriv_field0
3648
3649 #define MAX_ISP_TARG_TRANSFER   (2 << 20)
3650 #define NISP_TARG_CMDS          64
3651 #define NISP_TARG_NOTIFIES      64
3652 #define DISK_SHIFT              9
3653 #define JUNK_SIZE               256
3654 #define MULTI_CCB_DATA_LIM      8192
3655 //#define       MULTI_CCB_DATA_CNT      64
3656 #define MULTI_CCB_DATA_CNT      8
3657
3658 extern u_int vm_kmem_size;
3659 static int ca;
3660 static uint32_t disk_size;
3661 static uint8_t *disk_data = NULL;
3662 static uint8_t *junk_data;
3663 static MALLOC_DEFINE(M_ISPTARG, "ISPTARG", "ISP TARGET data");
3664 struct isptarg_softc {
3665         /* CCBs (CTIOs, ATIOs, INOTs) pending on the controller */
3666         struct isp_ccbq         work_queue;
3667         struct isp_ccbq         rework_queue;
3668         struct isp_ccbq         running_queue;
3669         struct isp_ccbq         inot_queue;
3670         struct cam_periph       *periph;
3671         struct cam_path         *path;
3672         ispsoftc_t              *isp;
3673 };
3674 static periph_ctor_t    isptargctor;
3675 static periph_dtor_t    isptargdtor;
3676 static periph_start_t   isptargstart;
3677 static periph_init_t    isptarginit;
3678 static void             isptarg_done(struct cam_periph *, union ccb *);
3679 static void             isptargasync(void *, u_int32_t, struct cam_path *, void *);
3680
3681
3682 static int isptarg_rwparm(uint8_t *, uint8_t *, uint64_t, uint32_t, uint8_t **, uint32_t *, int *);
3683
3684 static struct periph_driver isptargdriver =
3685 {
3686         isptarginit, "isptarg", TAILQ_HEAD_INITIALIZER(isptargdriver.units), 0
3687 };
3688
3689 static void
3690 isptarginit(void)
3691 {
3692 }
3693
3694 static void
3695 isptargnotify(ispsoftc_t *isp, union ccb *iccb, struct ccb_immediate_notify *inot)
3696 {
3697         struct ccb_notify_acknowledge *ack = &iccb->cna2;
3698
3699         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, inot->ccb_h.path, "%s: [0x%x] immediate notify for 0x%x from 0x%x status 0x%x arg 0x%x\n", __func__,
3700             inot->tag_id, inot->initiator_id, inot->seq_id, inot->ccb_h.status, inot->arg);
3701         ack->ccb_h.func_code = XPT_NOTIFY_ACKNOWLEDGE;
3702         ack->ccb_h.flags = 0;
3703         ack->ccb_h.retry_count = 0;
3704         ack->ccb_h.cbfcnp = isptarg_done;
3705         ack->ccb_h.timeout = 0;
3706         ack->ccb_h.ccb_inot = inot;
3707         ack->tag_id = inot->tag_id;
3708         ack->seq_id = inot->seq_id;
3709         ack->initiator_id = inot->initiator_id;
3710         xpt_action(iccb);
3711 }
3712
3713 static void
3714 isptargstart(struct cam_periph *periph, union ccb *iccb)
3715 {
3716         const uint8_t niliqd[SHORT_INQUIRY_LENGTH] = {
3717                 0x7f, 0x0, 0x5, 0x2, 32, 0, 0, 0x32,
3718                 'F', 'R', 'E', 'E', 'B', 'S', 'D', ' ',
3719                 'S', 'C', 'S', 'I', ' ', 'N', 'U', 'L',
3720                 'L', ' ', 'D', 'E', 'V', 'I', 'C', 'E',
3721                 '0', '0', '0', '1'
3722         };
3723         const uint8_t iqd[SHORT_INQUIRY_LENGTH] = {
3724                 0, 0x0, 0x5, 0x2, 32, 0, 0, 0x32,
3725                 'F', 'R', 'E', 'E', 'B', 'S', 'D', ' ',
3726                 'S', 'C', 'S', 'I', ' ', 'M', 'E', 'M',
3727                 'O', 'R', 'Y', ' ', 'D', 'I', 'S', 'K',
3728                 '0', '0', '0', '1'
3729         };
3730         int r, i, more = 0, last, is_data_cmd = 0, is_write;
3731         char *queue;
3732         struct isptarg_softc *softc = periph->softc;
3733         struct ccb_scsiio *csio;
3734         lun_id_t return_lun;
3735         struct ccb_accept_tio *atio;
3736         uint8_t *cdb, *ptr, status;
3737         uint8_t *data_ptr;
3738         uint32_t data_len, flags;
3739         struct ccb_hdr *ccbh;
3740
3741         mtx_assert(periph->sim->mtx, MA_OWNED);
3742         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG1, iccb->ccb_h.path, "%s: function code 0x%x INOTQ=%c WORKQ=%c REWORKQ=%c\n", __func__, iccb->ccb_h.func_code,
3743             TAILQ_FIRST(&softc->inot_queue)? 'y' : 'n', TAILQ_FIRST(&softc->work_queue)? 'y' : 'n', TAILQ_FIRST(&softc->rework_queue)? 'y' : 'n');
3744         /*
3745          * Check for immediate notifies first
3746          */
3747         ccbh = TAILQ_FIRST(&softc->inot_queue);
3748         if (ccbh) {
3749                 TAILQ_REMOVE(&softc->inot_queue, ccbh, periph_links.tqe);
3750                 if (TAILQ_FIRST(&softc->inot_queue) || TAILQ_FIRST(&softc->work_queue) || TAILQ_FIRST(&softc->rework_queue)) {
3751                         xpt_schedule(periph, 1);
3752                 }
3753                 isptargnotify(softc->isp, iccb, (struct ccb_immediate_notify *)ccbh);
3754                 return;
3755         }
3756
3757         /*
3758          * Check the rework (continuation) work queue first.
3759          */
3760         ccbh = TAILQ_FIRST(&softc->rework_queue);
3761         if (ccbh) {
3762                 atio = (struct ccb_accept_tio *)ccbh;
3763                 TAILQ_REMOVE(&softc->rework_queue, ccbh, periph_links.tqe);
3764                 more = TAILQ_FIRST(&softc->work_queue) || TAILQ_FIRST(&softc->rework_queue);
3765                 queue = "rework";
3766         } else {
3767                 ccbh = TAILQ_FIRST(&softc->work_queue);
3768                 if (ccbh == NULL) {
3769                         xpt_release_ccb(iccb);
3770                         return;
3771                 }
3772                 atio = (struct ccb_accept_tio *)ccbh;
3773                 TAILQ_REMOVE(&softc->work_queue, ccbh, periph_links.tqe);
3774                 more = TAILQ_FIRST(&softc->work_queue) != NULL;
3775                 queue = "work";
3776         }
3777         ATIO_PPD(atio)->on_queue = 0;
3778
3779         if (atio->tag_id == 0xffffffff || atio->ccb_h.func_code != XPT_ACCEPT_TARGET_IO) {
3780                 panic("BAD ATIO");
3781         }
3782
3783         data_len = is_write = 0;
3784         data_ptr = NULL;
3785         csio = &iccb->csio;
3786         status = SCSI_STATUS_OK;
3787         flags = CAM_SEND_STATUS;
3788         memset(&atio->sense_data, 0, sizeof (atio->sense_data));
3789         cdb = atio->cdb_io.cdb_bytes;
3790         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, ccbh->path, "%s: [0x%x] processing ATIO from %s queue initiator 0x%x CDB=0x%x data_offset=%u\n", __func__, atio->tag_id,
3791             queue, atio->init_id, cdb[0], ATIO_PPD(atio)->offset);
3792
3793         return_lun = XS_LUN(atio);
3794         if (return_lun != 0) {
3795                 xpt_print(atio->ccb_h.path, "[0x%x] Non-Zero Lun %d: cdb0=0x%x\n", atio->tag_id, return_lun, cdb[0]);
3796                 if (cdb[0] != INQUIRY && cdb[0] != REPORT_LUNS && cdb[0] != REQUEST_SENSE) {
3797                         status = SCSI_STATUS_CHECK_COND;
3798                         SDFIXED(atio->sense_data)->error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR;
3799                         SDFIXED(atio->sense_data)->flags = SSD_KEY_ILLEGAL_REQUEST;
3800                         SDFIXED(atio->sense_data)->add_sense_code = 0x25;       /* LOGICAL UNIT NOT SUPPORTED */
3801                         atio->sense_len = SSD_MIN_SIZE;
3802                 }
3803                 return_lun = CAM_LUN_WILDCARD;
3804         }
3805
3806         switch (cdb[0]) {
3807         case REQUEST_SENSE:
3808                 flags |= CAM_DIR_IN;
3809                 data_len = sizeof (atio->sense_data);
3810                 junk_data[0] = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR|SSD_KEY_NO_SENSE;
3811                 memset(junk_data+1, 0, data_len-1);
3812                 if (data_len > cdb[4]) {
3813                         data_len = cdb[4];
3814                 }
3815                 if (data_len) {
3816                         data_ptr = junk_data;
3817                 }
3818                 break;
3819         case WRITE_6:
3820         case WRITE_10:
3821         case WRITE_12:
3822         case WRITE_16:
3823                 is_write = 1;
3824                 /* FALLTHROUGH */
3825         case READ_6:
3826         case READ_10:
3827         case READ_12:
3828         case READ_16:
3829                 is_data_cmd = 1;
3830                 r = isptarg_rwparm(cdb, disk_data, disk_size, ATIO_PPD(atio)->offset, &data_ptr, &data_len, &last);
3831                 if (r != 0) {
3832                         status = SCSI_STATUS_CHECK_COND;
3833                         SDFIXED(atio->sense_data)->error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR;
3834                         SDFIXED(atio->sense_data)->flags = SSD_KEY_ILLEGAL_REQUEST;
3835                         if (r == -1) {
3836                                 SDFIXED(atio->sense_data)->add_sense_code = 0x21;       /* LOGICAL BLOCK ADDRESS OUT OF RANGE */
3837                         } else {
3838                                 SDFIXED(atio->sense_data)->add_sense_code = 0x20;       /* INVALID COMMAND OPERATION CODE */
3839                         }
3840                         atio->sense_len = SSD_MIN_SIZE;
3841                 } else {
3842 #ifdef  ISP_SEPARATE_STATUS
3843                         if (last && data_len) {
3844                                 last = 0;
3845                         }
3846 #endif
3847                         if (last == 0) {
3848                                 flags &= ~CAM_SEND_STATUS;
3849                         }
3850                         if (data_len) {
3851                                 ATIO_PPD(atio)->offset += data_len;
3852                                 if (is_write)
3853                                         flags |= CAM_DIR_OUT;
3854                                 else
3855                                         flags |= CAM_DIR_IN;
3856                         } else {
3857                                 flags |= CAM_DIR_NONE;
3858                         }
3859                 }
3860                 break;
3861         case INQUIRY:
3862                 flags |= CAM_DIR_IN;
3863                 if (cdb[1] || cdb[2] || cdb[3]) {
3864                         status = SCSI_STATUS_CHECK_COND;
3865                         SDFIXED(atio->sense_data)->error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR;
3866                         SDFIXED(atio->sense_data)->flags = SSD_KEY_UNIT_ATTENTION;
3867                         SDFIXED(atio->sense_data)->add_sense_code = 0x24;       /* INVALID FIELD IN CDB */
3868                         atio->sense_len = SSD_MIN_SIZE;
3869                         break;
3870                 }
3871                 data_len = sizeof (iqd);
3872                 if (data_len > cdb[4]) {
3873                         data_len = cdb[4];
3874                 }
3875                 if (data_len) {
3876                         if (XS_LUN(iccb) != 0) {
3877                                 memcpy(junk_data, niliqd, sizeof (iqd));
3878                         } else {
3879                                 memcpy(junk_data, iqd, sizeof (iqd));
3880                         }
3881                         data_ptr = junk_data;
3882                 }
3883                 break;
3884         case TEST_UNIT_READY:
3885                 flags |= CAM_DIR_NONE;
3886                 if (ca) {
3887                         ca = 0;
3888                         status = SCSI_STATUS_CHECK_COND;
3889                         SDFIXED(atio->sense_data)->error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR;
3890                         SDFIXED(atio->sense_data)->flags = SSD_KEY_UNIT_ATTENTION;
3891                         SDFIXED(atio->sense_data)->add_sense_code = 0x29;       /* POWER ON, RESET, OR BUS DEVICE RESET OCCURRED */
3892                         atio->sense_len = SSD_MIN_SIZE;
3893                 }
3894                 break;
3895         case SYNCHRONIZE_CACHE:
3896         case START_STOP:
3897         case RESERVE:
3898         case RELEASE:
3899         case VERIFY_10:
3900                 flags |= CAM_DIR_NONE;
3901                 break;
3902
3903         case READ_CAPACITY:
3904                 flags |= CAM_DIR_IN;
3905                 if (cdb[2] || cdb[3] || cdb[4] || cdb[5]) {
3906                         status = SCSI_STATUS_CHECK_COND;
3907                         SDFIXED(atio->sense_data)->error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR;
3908                         SDFIXED(atio->sense_data)->flags = SSD_KEY_ILLEGAL_REQUEST;
3909                         SDFIXED(atio->sense_data)->add_sense_code =  0x24;      /* INVALID FIELD IN CDB */
3910                         atio->sense_len = SSD_MIN_SIZE;
3911                         break;
3912                 }
3913                 if (cdb[8] & 0x1) { /* PMI */
3914                         junk_data[0] = 0xff;
3915                         junk_data[1] = 0xff;
3916                         junk_data[2] = 0xff;
3917                         junk_data[3] = 0xff;
3918                 } else {
3919                         uint64_t last_blk = (disk_size >> DISK_SHIFT) - 1;
3920                         if (last_blk < 0xffffffffULL) {
3921                             junk_data[0] = (last_blk >> 24) & 0xff;
3922                             junk_data[1] = (last_blk >> 16) & 0xff;
3923                             junk_data[2] = (last_blk >>  8) & 0xff;
3924                             junk_data[3] = (last_blk) & 0xff;
3925                         } else {
3926                             junk_data[0] = 0xff;
3927                             junk_data[1] = 0xff;
3928                             junk_data[2] = 0xff;
3929                             junk_data[3] = 0xff;
3930                         }
3931                 }
3932                 junk_data[4] = ((1 << DISK_SHIFT) >> 24) & 0xff;
3933                 junk_data[5] = ((1 << DISK_SHIFT) >> 16) & 0xff;
3934                 junk_data[6] = ((1 << DISK_SHIFT) >>  8) & 0xff;
3935                 junk_data[7] = ((1 << DISK_SHIFT)) & 0xff;
3936                 data_ptr = junk_data;
3937                 data_len = 8;
3938                 break;
3939         case REPORT_LUNS:
3940                 flags |= CAM_DIR_IN;
3941                 memset(junk_data, 0, JUNK_SIZE);
3942                 junk_data[0] = (1 << 3) >> 24;
3943                 junk_data[1] = (1 << 3) >> 16;
3944                 junk_data[2] = (1 << 3) >> 8;
3945                 junk_data[3] = (1 << 3);
3946                 ptr = NULL;
3947                 for (i = 0; i < 1; i++) {
3948                         ptr = &junk_data[8 + (i << 3)];
3949                         if (i >= 256) {
3950                                 ptr[0] = 0x40 | ((i >> 8) & 0x3f);
3951                         }
3952                         ptr[1] = i;
3953                 }
3954                 data_ptr = junk_data;
3955                 data_len = (ptr + 8) - junk_data;
3956                 break;
3957
3958         default:
3959                 flags |= CAM_DIR_NONE;
3960                 status = SCSI_STATUS_CHECK_COND;
3961                 SDFIXED(atio->sense_data)->error_code = SSD_ERRCODE_VALID|SSD_CURRENT_ERROR;
3962                 SDFIXED(atio->sense_data)->flags = SSD_KEY_ILLEGAL_REQUEST;
3963                 SDFIXED(atio->sense_data)->add_sense_code = 0x20;       /* INVALID COMMAND OPERATION CODE */
3964                 atio->sense_len = SSD_MIN_SIZE;
3965                 break;
3966         }
3967
3968         /*
3969          * If we are done with the transaction, tell the
3970          * controller to send status and perform a CMD_CMPLT.
3971          * If we have associated sense data, see if we can
3972          * send that too.
3973          */
3974         if (status == SCSI_STATUS_CHECK_COND) {
3975                 flags |= CAM_SEND_SENSE;
3976                 csio->sense_len = atio->sense_len;
3977                 csio->sense_data = atio->sense_data;
3978                 flags &= ~CAM_DIR_MASK;
3979                 data_len = 0;
3980                 data_ptr = NULL;
3981         }
3982         cam_fill_ctio(csio, 0, isptarg_done, flags, MSG_SIMPLE_Q_TAG, atio->tag_id, atio->init_id, status, data_ptr, data_len, 30 * hz);
3983         iccb->ccb_h.target_id = atio->ccb_h.target_id;
3984         iccb->ccb_h.target_lun = return_lun;
3985         iccb->ccb_h.ccb_atio = atio;
3986         CTIO_SEQ(iccb) = ATIO_PPD(atio)->sequence++;
3987         ATIO_PPD(atio)->ctio_cnt++;
3988         if (flags & CAM_SEND_STATUS) {
3989                 KASSERT((ATIO_PPD(atio)->status_sent == 0), ("we have already sent status for 0x%x in %s", atio->tag_id, __func__));
3990                 ATIO_PPD(atio)->status_sent = 1;
3991         }
3992         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, atio->ccb_h.path, "%s: sending downstream for  0x%x sequence %u len %u flags %x\n", __func__, atio->tag_id, CTIO_SEQ(iccb), data_len, flags);
3993         xpt_action(iccb);
3994
3995         if ((atio->ccb_h.status & CAM_DEV_QFRZN) != 0) {
3996                 cam_release_devq(periph->path, 0, 0, 0, 0);
3997                 atio->ccb_h.status &= ~CAM_DEV_QFRZN;
3998         }
3999 #ifdef  ISP_MULTI_CCBS
4000         if (is_data_cmd && ATIO_PPD(atio)->status_sent == 0 && ATIO_PPD(atio)->ctio_cnt < MULTI_CCB_DATA_CNT && ATIO_PPD(atio)->on_queue == 0) {
4001                 ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG0, atio->ccb_h.path, "%s: more still to do for 0x%x\n", __func__, atio->tag_id);
4002                 TAILQ_INSERT_TAIL(&softc->rework_queue, &atio->ccb_h, periph_links.tqe);
4003                 ATIO_PPD(atio)->on_queue = 1;
4004                 more = 1;
4005         }
4006 #endif
4007         if (more) {
4008                 xpt_schedule(periph, 1);
4009         }
4010 }
4011
4012 static cam_status
4013 isptargctor(struct cam_periph *periph, void *arg)
4014 {
4015         struct isptarg_softc *softc;
4016
4017         softc = (struct isptarg_softc *)arg;
4018         periph->softc = softc;
4019         softc->periph = periph;
4020         softc->path = periph->path;
4021         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG1, periph->path, "%s called\n", __func__);
4022         return (CAM_REQ_CMP);
4023 }
4024
4025 static void
4026 isptargdtor(struct cam_periph *periph)
4027 {
4028         struct isptarg_softc *softc;
4029         softc = (struct isptarg_softc *)periph->softc;
4030         ISP_PATH_PRT(softc->isp, ISP_LOGTDEBUG1, periph->path, "%s called\n", __func__);
4031         softc->periph = NULL;
4032         softc->path = NULL;
4033         periph->softc = NULL;
4034 }
4035
4036 static void
4037 isptarg_done(struct cam_periph *periph, union ccb *ccb)
4038 {
4039         struct isptarg_softc *softc;
4040         ispsoftc_t *isp;
4041         uint32_t newoff;
4042         struct ccb_accept_tio *atio;
4043         struct ccb_immediate_notify *inot;
4044         cam_status status;
4045
4046         softc = (struct isptarg_softc *)periph->softc;
4047         isp = softc->isp;
4048         status = ccb->ccb_h.status & CAM_STATUS_MASK;
4049
4050         switch (ccb->ccb_h.func_code) {
4051         case XPT_ACCEPT_TARGET_IO:
4052                 atio = (struct ccb_accept_tio *) ccb;
4053                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] ATIO seen in %s\n", atio->tag_id, __func__);
4054                 memset(ATIO_PPD(atio), 0, sizeof (ppd_t));
4055                 TAILQ_INSERT_TAIL(&softc->work_queue, &ccb->ccb_h, periph_links.tqe);
4056                 ATIO_PPD(atio)->on_queue = 1;
4057                 xpt_schedule(periph, 1);
4058                 break;
4059         case XPT_IMMEDIATE_NOTIFY:
4060                 inot = (struct ccb_immediate_notify *) ccb;
4061                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] INOT for 0x%x seen in %s\n", inot->tag_id, inot->seq_id, __func__);
4062                 TAILQ_INSERT_TAIL(&softc->inot_queue, &ccb->ccb_h, periph_links.tqe);
4063                 xpt_schedule(periph, 1);
4064                 break;
4065         case XPT_CONT_TARGET_IO:
4066                 atio = ccb->ccb_h.ccb_atio;
4067                 KASSERT((ATIO_PPD(atio)->ctio_cnt != 0), ("ctio zero when finishing a CTIO"));
4068                 ATIO_PPD(atio)->ctio_cnt--;
4069                 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
4070                         switch (ccb->ccb_h.status & CAM_STATUS_MASK) {
4071                         case CAM_MESSAGE_RECV:
4072                                 newoff = (ccb->csio.msg_ptr[3] << 24) | (ccb->csio.msg_ptr[4] << 16) | (ccb->csio.msg_ptr[5] << 8) | (ccb->csio.msg_ptr[6]);
4073                                 ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "[0x%x] got message to return to reset offset to 0x%x at sequence %u\n", atio->tag_id, newoff, CTIO_SEQ(ccb));
4074                                 ATIO_PPD(atio)->offset = newoff;
4075                                 ATIO_PPD(atio)->status_sent = 0;
4076                                 if (ATIO_PPD(atio)->on_queue == 0) {
4077                                         TAILQ_INSERT_TAIL(&softc->rework_queue, &atio->ccb_h, periph_links.tqe);
4078                                         ATIO_PPD(atio)->on_queue = 1;
4079                                 }
4080                                 xpt_schedule(periph, 1);
4081                                 break;
4082                         default:
4083                                 cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
4084                                 xpt_action((union ccb *)atio);
4085                                 break;
4086                         }
4087                 } else if ((ccb->ccb_h.flags & CAM_SEND_STATUS) == 0) {
4088                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] MID CTIO sequence %u seen in %s\n", atio->tag_id, CTIO_SEQ(ccb), __func__);
4089                         if (ATIO_PPD(atio)->status_sent == 0 && ATIO_PPD(atio)->on_queue == 0) {
4090                                 TAILQ_INSERT_TAIL(&softc->rework_queue, &atio->ccb_h, periph_links.tqe);
4091                                 ATIO_PPD(atio)->on_queue = 1;
4092                         }
4093                         xpt_schedule(periph, 1);
4094                 } else {
4095                         KASSERT((ATIO_PPD(atio)->ctio_cnt == 0), ("ctio count still %d when we think we've sent the STATUS ctio", ATIO_PPD(atio)->ctio_cnt));
4096                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "[0x%x] FINAL CTIO sequence %u seen in %s\n", atio->tag_id, CTIO_SEQ(ccb), __func__);
4097                         xpt_action((union ccb *)atio);
4098                 }
4099                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4100                         cam_release_devq(ccb->ccb_h.path, 0, 0, 0, 0);
4101                         ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
4102                 }
4103                 xpt_release_ccb(ccb);
4104                 break;
4105         case XPT_NOTIFY_ACKNOWLEDGE:
4106                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
4107                         cam_release_devq(ccb->ccb_h.path, 0, 0, 0, 0);
4108                         ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
4109                 }
4110                 inot = ccb->ccb_h.ccb_inot;
4111                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG1, inot->ccb_h.path, "[0x%x] recycle notify for tag 0x%x\n", inot->tag_id, inot->seq_id);
4112                 xpt_release_ccb(ccb);
4113                 xpt_action((union ccb *)inot);
4114                 break;
4115         default:
4116                 xpt_print(ccb->ccb_h.path, "unexpected code 0x%x\n", ccb->ccb_h.func_code);
4117                 break;
4118         }
4119 }
4120
4121 static void
4122 isptargasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg)
4123 {
4124         struct ac_contract *acp = arg;
4125         struct ac_device_changed *fc = (struct ac_device_changed *) acp->contract_data;
4126
4127         if (code != AC_CONTRACT) {
4128                 return;
4129         }
4130         xpt_print(path, "0x%016llx Port ID 0x%06x %s\n", (unsigned long long) fc->wwpn, fc->port, fc->arrived? "arrived" : "departed");
4131 }
4132
4133 static void
4134 isp_target_thread(ispsoftc_t *isp, int chan)
4135 {
4136         union ccb *ccb = NULL;
4137         int i;
4138         void *wchan;
4139         cam_status status;
4140         struct isptarg_softc *softc = NULL;
4141         struct cam_periph *periph = NULL, *wperiph = NULL;
4142         struct cam_path *path, *wpath;
4143         struct cam_sim *sim;
4144
4145         if (disk_data == NULL) {
4146                 disk_size = roundup2(vm_kmem_size >> 1, (1ULL << 20));
4147                 if (disk_size < (50 << 20)) {
4148                         disk_size = 50 << 20;
4149                 }
4150                 disk_data = malloc(disk_size, M_ISPTARG, M_WAITOK | M_ZERO);
4151                 if (disk_data == NULL) {
4152                         isp_prt(isp, ISP_LOGERR, "%s: could not allocate disk data", __func__);
4153                         goto out;
4154                 }
4155                 isp_prt(isp, ISP_LOGINFO, "allocated a %ju MiB disk", (uintmax_t) (disk_size >> 20));
4156         }
4157         junk_data = malloc(JUNK_SIZE, M_ISPTARG, M_WAITOK | M_ZERO);
4158         if (junk_data == NULL) {
4159                 isp_prt(isp, ISP_LOGERR, "%s: could not allocate junk", __func__);
4160                 goto out;
4161         }
4162
4163
4164         softc = malloc(sizeof (*softc), M_ISPTARG, M_WAITOK | M_ZERO);
4165         if (softc == NULL) {
4166                 isp_prt(isp, ISP_LOGERR, "%s: could not allocate softc", __func__);
4167                 goto out;
4168         }
4169         TAILQ_INIT(&softc->work_queue);
4170         TAILQ_INIT(&softc->rework_queue);
4171         TAILQ_INIT(&softc->running_queue);
4172         TAILQ_INIT(&softc->inot_queue);
4173         softc->isp = isp;
4174
4175         periphdriver_register(&isptargdriver);
4176         ISP_GET_PC(isp, chan, sim, sim);
4177         ISP_GET_PC(isp, chan, path,  path);
4178         status = xpt_create_path(&wpath, NULL, cam_sim_path(sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
4179         if (status != CAM_REQ_CMP) {
4180                 isp_prt(isp, ISP_LOGERR, "%s: could not allocate wildcard path", __func__);
4181                 return;
4182         }
4183         status = xpt_create_path(&path, NULL, cam_sim_path(sim), 0, 0);
4184         if (status != CAM_REQ_CMP) {
4185                 xpt_free_path(wpath);
4186                 isp_prt(isp, ISP_LOGERR, "%s: could not allocate path", __func__);
4187                 return;
4188         }
4189
4190         ISP_LOCK(isp);
4191         status = cam_periph_alloc(isptargctor, NULL, isptargdtor, isptargstart, "isptarg", CAM_PERIPH_BIO, wpath, NULL, 0, softc);
4192         if (status != CAM_REQ_CMP) {
4193                 ISP_UNLOCK(isp);
4194                 isp_prt(isp, ISP_LOGERR, "%s: cam_periph_alloc for wildcard failed", __func__);
4195                 goto out;
4196         }
4197         wperiph = cam_periph_find(wpath, "isptarg");
4198         if (wperiph == NULL) {
4199                 ISP_UNLOCK(isp);
4200                 isp_prt(isp, ISP_LOGERR, "%s: wildcard periph already allocated but doesn't exist", __func__);
4201                 goto out;
4202         }
4203
4204         status = cam_periph_alloc(isptargctor, NULL, isptargdtor, isptargstart, "isptarg", CAM_PERIPH_BIO, path, NULL, 0, softc);
4205         if (status != CAM_REQ_CMP) {
4206                 ISP_UNLOCK(isp);
4207                 isp_prt(isp, ISP_LOGERR, "%s: cam_periph_alloc failed", __func__);
4208                 goto out;
4209         }
4210
4211         periph = cam_periph_find(path, "isptarg");
4212         if (periph == NULL) {
4213                 ISP_UNLOCK(isp);
4214                 isp_prt(isp, ISP_LOGERR, "%s: periph already allocated but doesn't exist", __func__);
4215                 goto out;
4216         }
4217
4218         status = xpt_register_async(AC_CONTRACT, isptargasync, isp, wpath);
4219         if (status != CAM_REQ_CMP) {
4220                 ISP_UNLOCK(isp);
4221                 isp_prt(isp, ISP_LOGERR, "%s: xpt_register_async failed", __func__);
4222                 goto out;
4223         }
4224
4225         ISP_UNLOCK(isp);
4226
4227         ccb = xpt_alloc_ccb();
4228
4229         /*
4230          * Make sure role is none.
4231          */
4232         xpt_setup_ccb(&ccb->ccb_h, periph->path, 10);
4233         ccb->ccb_h.func_code = XPT_SET_SIM_KNOB;
4234         ccb->knob.xport_specific.fc.role = KNOB_ROLE_NONE;
4235         ccb->knob.xport_specific.fc.valid = KNOB_VALID_ROLE;
4236
4237         ISP_LOCK(isp);
4238         xpt_action(ccb);
4239         ISP_UNLOCK(isp);
4240
4241         /*
4242          * Now enable luns
4243          */
4244         xpt_setup_ccb(&ccb->ccb_h, periph->path, 10);
4245         ccb->ccb_h.func_code = XPT_EN_LUN;
4246         ccb->cel.enable = 1;
4247         ISP_LOCK(isp);
4248         xpt_action(ccb);
4249         ISP_UNLOCK(isp);
4250         if (ccb->ccb_h.status != CAM_REQ_CMP) {
4251                 xpt_free_ccb(ccb);
4252                 xpt_print(periph->path, "failed to enable lun (0x%x)\n", ccb->ccb_h.status);
4253                 goto out;
4254         }
4255
4256         xpt_setup_ccb(&ccb->ccb_h, wperiph->path, 10);
4257         ccb->ccb_h.func_code = XPT_EN_LUN;
4258         ccb->cel.enable = 1;
4259         ISP_LOCK(isp);
4260         xpt_action(ccb);
4261         ISP_UNLOCK(isp);
4262         if (ccb->ccb_h.status != CAM_REQ_CMP) {
4263                 xpt_free_ccb(ccb);
4264                 xpt_print(wperiph->path, "failed to enable lun (0x%x)\n", ccb->ccb_h.status);
4265                 goto out;
4266         }
4267         xpt_free_ccb(ccb);
4268
4269         /*
4270          * Add resources
4271          */
4272         ISP_GET_PC_ADDR(isp, chan, target_proc, wchan);
4273         for (i = 0; i < 4; i++) {
4274                 ccb = malloc(sizeof (*ccb), M_ISPTARG, M_WAITOK | M_ZERO);
4275                 xpt_setup_ccb(&ccb->ccb_h, wperiph->path, 1);
4276                 ccb->ccb_h.func_code = XPT_ACCEPT_TARGET_IO;
4277                 ccb->ccb_h.cbfcnp = isptarg_done;
4278                 ccb->ccb_h.ppriv_ptr0 = malloc(sizeof (ppd_t), M_ISPTARG, M_WAITOK | M_ZERO);
4279                 ISP_LOCK(isp);
4280                 xpt_action(ccb);
4281                 ISP_UNLOCK(isp);
4282         }
4283         for (i = 0; i < NISP_TARG_CMDS; i++) {
4284                 ccb = malloc(sizeof (*ccb), M_ISPTARG, M_WAITOK | M_ZERO);
4285                 xpt_setup_ccb(&ccb->ccb_h, periph->path, 1);
4286                 ccb->ccb_h.func_code = XPT_ACCEPT_TARGET_IO;
4287                 ccb->ccb_h.cbfcnp = isptarg_done;
4288                 ccb->ccb_h.ppriv_ptr0 = malloc(sizeof (ppd_t), M_ISPTARG, M_WAITOK | M_ZERO);
4289                 ISP_LOCK(isp);
4290                 xpt_action(ccb);
4291                 ISP_UNLOCK(isp);
4292         }
4293         for (i = 0; i < 4; i++) {
4294                 ccb = malloc(sizeof (*ccb), M_ISPTARG, M_WAITOK | M_ZERO);
4295                 xpt_setup_ccb(&ccb->ccb_h, wperiph->path, 1);
4296                 ccb->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY;
4297                 ccb->ccb_h.cbfcnp = isptarg_done;
4298                 ISP_LOCK(isp);
4299                 xpt_action(ccb);
4300                 ISP_UNLOCK(isp);
4301         }
4302         for (i = 0; i < NISP_TARG_NOTIFIES; i++) {
4303                 ccb = malloc(sizeof (*ccb), M_ISPTARG, M_WAITOK | M_ZERO);
4304                 xpt_setup_ccb(&ccb->ccb_h, periph->path, 1);
4305                 ccb->ccb_h.func_code = XPT_IMMEDIATE_NOTIFY;
4306                 ccb->ccb_h.cbfcnp = isptarg_done;
4307                 ISP_LOCK(isp);
4308                 xpt_action(ccb);
4309                 ISP_UNLOCK(isp);
4310         }
4311
4312         /*
4313          * Now turn it all back on
4314          */
4315         xpt_setup_ccb(&ccb->ccb_h, periph->path, 10);
4316         ccb->ccb_h.func_code = XPT_SET_SIM_KNOB;
4317         ccb->knob.xport_specific.fc.valid = KNOB_VALID_ROLE;
4318         ccb->knob.xport_specific.fc.role = KNOB_ROLE_TARGET;
4319         ISP_LOCK(isp);
4320         xpt_action(ccb);
4321         ISP_UNLOCK(isp);
4322
4323         /*
4324          * Okay, while things are still active, sleep...
4325          */
4326         ISP_LOCK(isp);
4327         for (;;) {
4328                 ISP_GET_PC(isp, chan, proc_active, i);
4329                 if (i == 0) {
4330                         break;
4331                 }
4332                 msleep(wchan, &isp->isp_lock, PUSER, "tsnooze", 0);
4333         }
4334         ISP_UNLOCK(isp);
4335
4336 out:
4337         if (wperiph) {
4338                 cam_periph_invalidate(wperiph);
4339         }
4340         if (periph) {
4341                 cam_periph_invalidate(periph);
4342         }
4343         if (junk_data) {
4344                 free(junk_data, M_ISPTARG);
4345         }
4346         if (disk_data) {
4347                 free(disk_data, M_ISPTARG);
4348         }
4349         if (softc) {
4350                 free(softc, M_ISPTARG);
4351         }
4352         xpt_free_path(path);
4353         xpt_free_path(wpath);
4354 }
4355
4356 static void
4357 isp_target_thread_pi(void *arg)
4358 {
4359         struct isp_spi *pi = arg;
4360         isp_target_thread(cam_sim_softc(pi->sim), cam_sim_bus(pi->sim));
4361 }
4362
4363 static void
4364 isp_target_thread_fc(void *arg)
4365 {
4366         struct isp_fc *fc = arg;
4367         isp_target_thread(cam_sim_softc(fc->sim), cam_sim_bus(fc->sim));
4368 }
4369
4370 static int
4371 isptarg_rwparm(uint8_t *cdb, uint8_t *dp, uint64_t dl, uint32_t offset, uint8_t **kp, uint32_t *tl, int *lp)
4372 {
4373         uint32_t cnt, curcnt;
4374         uint64_t lba;
4375
4376         switch (cdb[0]) {
4377         case WRITE_16:
4378         case READ_16:
4379                 cnt =   (((uint32_t)cdb[10]) <<  24) |
4380                         (((uint32_t)cdb[11]) <<  16) |
4381                         (((uint32_t)cdb[12]) <<   8) |
4382                         ((uint32_t)cdb[13]);
4383
4384                 lba =   (((uint64_t)cdb[2]) << 56) |
4385                         (((uint64_t)cdb[3]) << 48) |
4386                         (((uint64_t)cdb[4]) << 40) |
4387                         (((uint64_t)cdb[5]) << 32) |
4388                         (((uint64_t)cdb[6]) << 24) |
4389                         (((uint64_t)cdb[7]) << 16) |
4390                         (((uint64_t)cdb[8]) <<  8) |
4391                         ((uint64_t)cdb[9]);
4392                 break;
4393         case WRITE_12:
4394         case READ_12:
4395                 cnt =   (((uint32_t)cdb[6]) <<  16) |
4396                         (((uint32_t)cdb[7]) <<   8) |
4397                         ((u_int32_t)cdb[8]);
4398
4399                 lba =   (((uint32_t)cdb[2]) << 24) |
4400                         (((uint32_t)cdb[3]) << 16) |
4401                         (((uint32_t)cdb[4]) <<  8) |
4402                         ((uint32_t)cdb[5]);
4403                 break;
4404         case WRITE_10:
4405         case READ_10:
4406                 cnt =   (((uint32_t)cdb[7]) <<  8) |
4407                         ((u_int32_t)cdb[8]);
4408
4409                 lba =   (((uint32_t)cdb[2]) << 24) |
4410                         (((uint32_t)cdb[3]) << 16) |
4411                         (((uint32_t)cdb[4]) <<  8) |
4412                         ((uint32_t)cdb[5]);
4413                 break;
4414         case WRITE_6:
4415         case READ_6:
4416                 cnt = cdb[4];
4417                 if (cnt == 0) {
4418                         cnt = 256;
4419                 }
4420                 lba =   (((uint32_t)cdb[1] & 0x1f) << 16) |
4421                         (((uint32_t)cdb[2]) << 8) |
4422                         ((uint32_t)cdb[3]);
4423                 break;
4424         default:
4425                 return (-1);
4426         }
4427
4428         cnt <<= DISK_SHIFT;
4429         lba <<= DISK_SHIFT;
4430
4431         if (offset == cnt) {
4432                 *lp = 1;
4433                 return (0);
4434         }
4435
4436         if (lba + cnt > dl) {
4437                 return (-2);
4438         }
4439
4440         curcnt = MAX_ISP_TARG_TRANSFER;
4441         if (offset + curcnt >= cnt) {
4442                 curcnt = cnt - offset;
4443                 *lp = 1;
4444         } else {
4445                 *lp = 0;
4446         }
4447 #ifdef  ISP_MULTI_CCBS
4448         if (curcnt > MULTI_CCB_DATA_LIM)
4449                 curcnt = MULTI_CCB_DATA_LIM;
4450 #endif
4451         *tl = curcnt;
4452         *kp = &dp[lba + offset];
4453         return (0);
4454 }
4455
4456 #endif
4457 #endif
4458
4459 static void
4460 isp_cam_async(void *cbarg, uint32_t code, struct cam_path *path, void *arg)
4461 {
4462         struct cam_sim *sim;
4463         int bus, tgt;
4464         ispsoftc_t *isp;
4465
4466         sim = (struct cam_sim *)cbarg;
4467         isp = (ispsoftc_t *) cam_sim_softc(sim);
4468         bus = cam_sim_bus(sim);
4469         tgt = xpt_path_target_id(path);
4470
4471         switch (code) {
4472         case AC_LOST_DEVICE:
4473                 if (IS_SCSI(isp)) {
4474                         uint16_t oflags, nflags;
4475                         sdparam *sdp = SDPARAM(isp, bus);
4476
4477                         if (tgt >= 0) {
4478                                 nflags = sdp->isp_devparam[tgt].nvrm_flags;
4479 #ifndef ISP_TARGET_MODE
4480                                 nflags &= DPARM_SAFE_DFLT;
4481                                 if (isp->isp_loaded_fw) {
4482                                         nflags |= DPARM_NARROW | DPARM_ASYNC;
4483                                 }
4484 #else
4485                                 nflags = DPARM_DEFAULT;
4486 #endif
4487                                 oflags = sdp->isp_devparam[tgt].goal_flags;
4488                                 sdp->isp_devparam[tgt].goal_flags = nflags;
4489                                 sdp->isp_devparam[tgt].dev_update = 1;
4490                                 sdp->update = 1;
4491                                 (void) isp_control(isp, ISPCTL_UPDATE_PARAMS, bus);
4492                                 sdp->isp_devparam[tgt].goal_flags = oflags;
4493                         }
4494                 }
4495                 break;
4496         default:
4497                 isp_prt(isp, ISP_LOGWARN, "isp_cam_async: Code 0x%x", code);
4498                 break;
4499         }
4500 }
4501
4502 static void
4503 isp_poll(struct cam_sim *sim)
4504 {
4505         ispsoftc_t *isp = cam_sim_softc(sim);
4506         uint32_t isr;
4507         uint16_t sema, mbox;
4508
4509         if (ISP_READ_ISR(isp, &isr, &sema, &mbox)) {
4510                 isp_intr(isp, isr, sema, mbox);
4511         }
4512 }
4513
4514
4515 static void
4516 isp_watchdog(void *arg)
4517 {
4518         struct ccb_scsiio *xs = arg;
4519         ispsoftc_t *isp;
4520         uint32_t ohandle = ISP_HANDLE_FREE, handle;
4521
4522         isp = XS_ISP(xs);
4523
4524         handle = isp_find_handle(isp, xs);
4525
4526         /*
4527          * Hand crank the interrupt code just to be sure the command isn't stuck somewhere.
4528          */
4529         if (handle != ISP_HANDLE_FREE) {
4530                 uint32_t isr;
4531                 uint16_t sema, mbox;
4532                 if (ISP_READ_ISR(isp, &isr, &sema, &mbox) != 0) {
4533                         isp_intr(isp, isr, sema, mbox);
4534                 }
4535                 ohandle = handle;
4536                 handle = isp_find_handle(isp, xs);
4537         }
4538         if (handle != ISP_HANDLE_FREE) {
4539                 /*
4540                  * Try and make sure the command is really dead before
4541                  * we release the handle (and DMA resources) for reuse.
4542                  *
4543                  * If we are successful in aborting the command then
4544                  * we're done here because we'll get the command returned
4545                  * back separately.
4546                  */
4547                 if (isp_control(isp, ISPCTL_ABORT_CMD, xs) == 0) {
4548                         return;
4549                 }
4550
4551                 /*
4552                  * Note that after calling the above, the command may in
4553                  * fact have been completed.
4554                  */
4555                 xs = isp_find_xs(isp, handle);
4556
4557                 /*
4558                  * If the command no longer exists, then we won't
4559                  * be able to find the xs again with this handle.
4560                  */
4561                 if (xs == NULL) {
4562                         return;
4563                 }
4564
4565                 /*
4566                  * After this point, the command is really dead.
4567                  */
4568                 if (XS_XFRLEN(xs)) {
4569                         ISP_DMAFREE(isp, xs, handle);
4570                 }
4571                 isp_destroy_handle(isp, handle);
4572                 isp_prt(isp, ISP_LOGERR, "%s: timeout for handle 0x%x", __func__, handle);
4573                 xs->ccb_h.status &= ~CAM_STATUS_MASK;
4574                 xs->ccb_h.status |= CAM_CMD_TIMEOUT;
4575                 isp_prt_endcmd(isp, xs);
4576                 isp_done(xs);
4577         } else {
4578                 if (ohandle != ISP_HANDLE_FREE) {
4579                         isp_prt(isp, ISP_LOGWARN, "%s: timeout for handle 0x%x, recovered during interrupt", __func__, ohandle);
4580                 } else {
4581                         isp_prt(isp, ISP_LOGWARN, "%s: timeout for handle already free", __func__);
4582                 }
4583         }
4584 }
4585
4586 static void
4587 isp_make_here(ispsoftc_t *isp, fcportdb_t *fcp, int chan, int tgt)
4588 {
4589         union ccb *ccb;
4590         struct isp_fc *fc = ISP_FC_PC(isp, chan);
4591
4592         if (isp_autoconfig == 0) {
4593                 return;
4594         }
4595
4596         /*
4597          * Allocate a CCB, create a wildcard path for this target and schedule a rescan.
4598          */
4599         ccb = xpt_alloc_ccb_nowait();
4600         if (ccb == NULL) {
4601                 isp_prt(isp, ISP_LOGWARN, "Chan %d unable to alloc CCB for rescan", chan);
4602                 return;
4603         }
4604         if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(fc->sim),
4605             tgt, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
4606                 isp_prt(isp, ISP_LOGWARN, "unable to create path for rescan");
4607                 xpt_free_ccb(ccb);
4608                 return;
4609         }
4610
4611         /*
4612          * Since we're about to issue a rescan, mark this device as not
4613          * reported gone.
4614          */
4615         fcp->reported_gone = 0;
4616
4617         xpt_rescan(ccb);
4618 }
4619
4620 static void
4621 isp_make_gone(ispsoftc_t *isp, fcportdb_t *fcp, int chan, int tgt)
4622 {
4623         struct cam_path *tp;
4624         struct isp_fc *fc = ISP_FC_PC(isp, chan);
4625
4626         if (isp_autoconfig == 0) {
4627                 return;
4628         }
4629         if (xpt_create_path(&tp, NULL, cam_sim_path(fc->sim), tgt, CAM_LUN_WILDCARD) == CAM_REQ_CMP) {
4630                 /*
4631                  * We're about to send out the lost device async
4632                  * notification, so indicate that we have reported it gone.
4633                  */
4634                 fcp->reported_gone = 1;
4635                 xpt_async(AC_LOST_DEVICE, tp, NULL);
4636                 xpt_free_path(tp);
4637         }
4638 }
4639
4640 /*
4641  * Gone Device Timer Function- when we have decided that a device has gone
4642  * away, we wait a specific period of time prior to telling the OS it has
4643  * gone away.
4644  *
4645  * This timer function fires once a second and then scans the port database
4646  * for devices that are marked dead but still have a virtual target assigned.
4647  * We decrement a counter for that port database entry, and when it hits zero,
4648  * we tell the OS the device has gone away.
4649  */
4650 static void
4651 isp_gdt(void *arg)
4652 {
4653         struct isp_fc *fc = arg;
4654         taskqueue_enqueue(taskqueue_thread, &fc->gtask);
4655 }
4656
4657 static void
4658 isp_gdt_task(void *arg, int pending)
4659 {
4660         struct isp_fc *fc = arg;
4661         ispsoftc_t *isp = fc->isp;
4662         int chan = fc - isp->isp_osinfo.pc.fc;
4663         fcportdb_t *lp;
4664         int dbidx, tgt, more_to_do = 0;
4665
4666         ISP_LOCK(isp);
4667         isp_prt(isp, ISP_LOGDEBUG0, "Chan %d GDT timer expired", chan);
4668         for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
4669                 lp = &FCPARAM(isp, chan)->portdb[dbidx];
4670
4671                 if (lp->state != FC_PORTDB_STATE_ZOMBIE) {
4672                         continue;
4673                 }
4674                 if (lp->dev_map_idx == 0 || lp->target_mode) {
4675                         continue;
4676                 }
4677                 if (lp->gone_timer != 0) {
4678                         isp_prt(isp, ISP_LOG_SANCFG, "%s: Chan %d more to do for target %u (timer=%u)", __func__, chan, lp->dev_map_idx - 1, lp->gone_timer);
4679                         lp->gone_timer -= 1;
4680                         more_to_do++;
4681                         continue;
4682                 }
4683                 tgt = lp->dev_map_idx - 1;
4684                 FCPARAM(isp, chan)->isp_dev_map[tgt] = 0;
4685                 lp->dev_map_idx = 0;
4686                 lp->state = FC_PORTDB_STATE_NIL;
4687                 isp_prt(isp, ISP_LOGCONFIG, prom3, chan, lp->portid, tgt, "Gone Device Timeout");
4688                 isp_make_gone(isp, lp, chan, tgt);
4689         }
4690         if (fc->ready) {
4691                 if (more_to_do) {
4692                         callout_reset(&fc->gdt, hz, isp_gdt, fc);
4693                 } else {
4694                         callout_deactivate(&fc->gdt);
4695                         isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Stopping Gone Device Timer @ %lu", chan, (unsigned long) time_uptime);
4696                 }
4697         }
4698         ISP_UNLOCK(isp);
4699 }
4700
4701 /*
4702  * Loop Down Timer Function- when loop goes down, a timer is started and
4703  * and after it expires we come here and take all probational devices that
4704  * the OS knows about and the tell the OS that they've gone away.
4705  *
4706  * We don't clear the devices out of our port database because, when loop
4707  * come back up, we have to do some actual cleanup with the chip at that
4708  * point (implicit PLOGO, e.g., to get the chip's port database state right).
4709  */
4710 static void
4711 isp_ldt(void *arg)
4712 {
4713         struct isp_fc *fc = arg;
4714         taskqueue_enqueue(taskqueue_thread, &fc->ltask);
4715 }
4716
4717 static void
4718 isp_ldt_task(void *arg, int pending)
4719 {
4720         struct isp_fc *fc = arg;
4721         ispsoftc_t *isp = fc->isp;
4722         int chan = fc - isp->isp_osinfo.pc.fc;
4723         fcportdb_t *lp;
4724         int dbidx, tgt, i;
4725
4726         ISP_LOCK(isp);
4727         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Loop Down Timer expired @ %lu", chan, (unsigned long) time_uptime);
4728         callout_deactivate(&fc->ldt);
4729
4730         /*
4731          * Notify to the OS all targets who we now consider have departed.
4732          */
4733         for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
4734                 lp = &FCPARAM(isp, chan)->portdb[dbidx];
4735
4736                 if (lp->state != FC_PORTDB_STATE_PROBATIONAL) {
4737                         continue;
4738                 }
4739                 if (lp->dev_map_idx == 0 || lp->target_mode) {
4740                         continue;
4741                 }
4742
4743                 /*
4744                  * XXX: CLEAN UP AND COMPLETE ANY PENDING COMMANDS FIRST!
4745                  */
4746
4747
4748                 for (i = 0; i < isp->isp_maxcmds; i++) {
4749                         struct ccb_scsiio *xs;
4750
4751                         if (!ISP_VALID_HANDLE(isp, isp->isp_xflist[i].handle)) {
4752                                 continue;
4753                         }
4754                         if ((xs = isp->isp_xflist[i].cmd) == NULL) {
4755                                 continue;
4756                         }
4757                         if (dbidx != (FCPARAM(isp, chan)->isp_dev_map[XS_TGT(xs)] - 1)) {
4758                                 continue;
4759                         }
4760                         isp_prt(isp, ISP_LOGWARN, "command handle 0x%x for %d.%d.%d orphaned by loop down timeout",
4761                             isp->isp_xflist[i].handle, chan, XS_TGT(xs), XS_LUN(xs));
4762                 }
4763
4764                 /*
4765                  * Mark that we've announced that this device is gone....
4766                  */
4767                 lp->announced = 1;
4768
4769                 /*
4770                  * but *don't* change the state of the entry. Just clear
4771                  * any target id stuff and announce to CAM that the
4772                  * device is gone. This way any necessary PLOGO stuff
4773                  * will happen when loop comes back up.
4774                  */
4775
4776                 tgt = lp->dev_map_idx - 1;
4777                 FCPARAM(isp, chan)->isp_dev_map[tgt] = 0;
4778                 lp->dev_map_idx = 0;
4779                 lp->state = FC_PORTDB_STATE_NIL;
4780                 isp_prt(isp, ISP_LOGCONFIG, prom3, chan, lp->portid, tgt, "Loop Down Timeout");
4781                 isp_make_gone(isp, lp, chan, tgt);
4782         }
4783
4784         if (FCPARAM(isp, chan)->role & ISP_ROLE_INITIATOR) {
4785                 isp_unfreeze_loopdown(isp, chan);
4786         }
4787         /*
4788          * The loop down timer has expired. Wake up the kthread
4789          * to notice that fact (or make it false).
4790          */
4791         fc->loop_dead = 1;
4792         fc->loop_down_time = fc->loop_down_limit+1;
4793         wakeup(fc);
4794         ISP_UNLOCK(isp);
4795 }
4796
4797 static void
4798 isp_kthread(void *arg)
4799 {
4800         struct isp_fc *fc = arg;
4801         ispsoftc_t *isp = fc->isp;
4802         int chan = fc - isp->isp_osinfo.pc.fc;
4803         int slp = 0;
4804
4805         mtx_lock(&isp->isp_osinfo.lock);
4806
4807         for (;;) {
4808                 int lb, lim;
4809
4810                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d checking FC state", __func__, chan);
4811                 lb = isp_fc_runstate(isp, chan, 250000);
4812
4813                 /*
4814                  * Our action is different based upon whether we're supporting
4815                  * Initiator mode or not. If we are, we might freeze the simq
4816                  * when loop is down and set all sorts of different delays to
4817                  * check again.
4818                  *
4819                  * If not, we simply just wait for loop to come up.
4820                  */
4821                 if (lb && (FCPARAM(isp, chan)->role & ISP_ROLE_INITIATOR)) {
4822                         /*
4823                          * Increment loop down time by the last sleep interval
4824                          */
4825                         fc->loop_down_time += slp;
4826
4827                         if (lb < 0) {
4828                                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d FC loop not up (down count %d)", __func__, chan, fc->loop_down_time);
4829                         } else {
4830                                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d FC got to %d (down count %d)", __func__, chan, lb, fc->loop_down_time);
4831                         }
4832
4833                         /*
4834                          * If we've never seen loop up and we've waited longer
4835                          * than quickboot time, or we've seen loop up but we've
4836                          * waited longer than loop_down_limit, give up and go
4837                          * to sleep until loop comes up.
4838                          */
4839                         if (FCPARAM(isp, chan)->loop_seen_once == 0) {
4840                                 lim = isp_quickboot_time;
4841                         } else {
4842                                 lim = fc->loop_down_limit;
4843                         }
4844                         if (fc->loop_down_time >= lim) {
4845                                 isp_freeze_loopdown(isp, chan, "loop limit hit");
4846                                 slp = 0;
4847                         } else if (fc->loop_down_time < 10) {
4848                                 slp = 1;
4849                         } else if (fc->loop_down_time < 30) {
4850                                 slp = 5;
4851                         } else if (fc->loop_down_time < 60) {
4852                                 slp = 10;
4853                         } else if (fc->loop_down_time < 120) {
4854                                 slp = 20;
4855                         } else {
4856                                 slp = 30;
4857                         }
4858
4859                 } else if (lb) {
4860                         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d FC Loop Down", __func__, chan);
4861                         fc->loop_down_time += slp;
4862                         if (fc->loop_down_time > 300)
4863                                 slp = 0;
4864                         else
4865                                 slp = 60;
4866                 } else {
4867                         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d FC state OK", __func__, chan);
4868                         fc->loop_down_time = 0;
4869                         slp = 0;
4870                 }
4871
4872
4873                 /*
4874                  * If this is past the first loop up or the loop is dead and if we'd frozen the simq, unfreeze it
4875                  * now so that CAM can start sending us commands.
4876                  *
4877                  * If the FC state isn't okay yet, they'll hit that in isp_start which will freeze the queue again
4878                  * or kill the commands, as appropriate.
4879                  */
4880
4881                 if (FCPARAM(isp, chan)->loop_seen_once || fc->loop_dead) {
4882                         isp_unfreeze_loopdown(isp, chan);
4883                 }
4884
4885                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d sleep time %d", __func__, chan, slp);
4886
4887                 msleep(fc, &isp->isp_osinfo.lock, PRIBIO, "ispf", slp * hz);
4888
4889                 /*
4890                  * If slp is zero, we're waking up for the first time after
4891                  * things have been okay. In this case, we set a deferral state
4892                  * for all commands and delay hysteresis seconds before starting
4893                  * the FC state evaluation. This gives the loop/fabric a chance
4894                  * to settle.
4895                  */
4896                 if (slp == 0 && fc->hysteresis) {
4897                         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "%s: Chan %d sleep hysteresis ticks %d", __func__, chan, fc->hysteresis * hz);
4898                         mtx_unlock(&isp->isp_osinfo.lock);
4899                         pause("ispt", fc->hysteresis * hz);
4900                         mtx_lock(&isp->isp_osinfo.lock);
4901                 }
4902         }
4903         mtx_unlock(&isp->isp_osinfo.lock);
4904 }
4905
4906 static void
4907 isp_action(struct cam_sim *sim, union ccb *ccb)
4908 {
4909         int bus, tgt, ts, error, lim;
4910         ispsoftc_t *isp;
4911         struct ccb_trans_settings *cts;
4912
4913         CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("isp_action\n"));
4914
4915         isp = (ispsoftc_t *)cam_sim_softc(sim);
4916         mtx_assert(&isp->isp_lock, MA_OWNED);
4917         isp_prt(isp, ISP_LOGDEBUG2, "isp_action code %x", ccb->ccb_h.func_code);
4918         ISP_PCMD(ccb) = NULL;
4919
4920         if (isp->isp_state != ISP_RUNSTATE && ccb->ccb_h.func_code == XPT_SCSI_IO) {
4921                 isp_init(isp);
4922                 if (isp->isp_state != ISP_INITSTATE) {
4923                         /*
4924                          * Lie. Say it was a selection timeout.
4925                          */
4926                         ccb->ccb_h.status = CAM_SEL_TIMEOUT;
4927                         isp_done((struct ccb_scsiio *) ccb);
4928                         return;
4929                 }
4930                 isp->isp_state = ISP_RUNSTATE;
4931         }
4932
4933         switch (ccb->ccb_h.func_code) {
4934         case XPT_SCSI_IO:       /* Execute the requested I/O operation */
4935                 bus = XS_CHANNEL(ccb);
4936                 /*
4937                  * Do a couple of preliminary checks...
4938                  */
4939                 if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0) {
4940                         if ((ccb->ccb_h.flags & CAM_CDB_PHYS) != 0) {
4941                                 ccb->ccb_h.status = CAM_REQ_INVALID;
4942                                 isp_done((struct ccb_scsiio *) ccb);
4943                                 break;
4944                         }
4945                 }
4946                 ccb->csio.req_map = NULL;
4947 #ifdef  DIAGNOSTIC
4948                 if (ccb->ccb_h.target_id > (ISP_MAX_TARGETS(isp) - 1)) {
4949                         xpt_print(ccb->ccb_h.path, "invalid target\n");
4950                         ccb->ccb_h.status = CAM_PATH_INVALID;
4951                 } else if (ccb->ccb_h.target_lun > (ISP_MAX_LUNS(isp) - 1)) {
4952                         xpt_print(ccb->ccb_h.path, "invalid lun\n");
4953                         ccb->ccb_h.status = CAM_PATH_INVALID;
4954                 }
4955                 if (ccb->ccb_h.status == CAM_PATH_INVALID) {
4956                         xpt_done(ccb);
4957                         break;
4958                 }
4959 #endif
4960                 ccb->csio.scsi_status = SCSI_STATUS_OK;
4961                 if (isp_get_pcmd(isp, ccb)) {
4962                         isp_prt(isp, ISP_LOGWARN, "out of PCMDs");
4963                         cam_freeze_devq(ccb->ccb_h.path);
4964                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 250, 0);
4965                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
4966                         xpt_done(ccb);
4967                         break;
4968                 }
4969                 error = isp_start((XS_T *) ccb);
4970                 switch (error) {
4971                 case CMD_QUEUED:
4972                         ccb->ccb_h.status |= CAM_SIM_QUEUED;
4973                         if (ccb->ccb_h.timeout == CAM_TIME_INFINITY) {
4974                                 break;
4975                         }
4976                         ts = ccb->ccb_h.timeout;
4977                         if (ts == CAM_TIME_DEFAULT) {
4978                                 ts = 60*1000;
4979                         }
4980                         ts = isp_mstohz(ts);
4981                         callout_reset(&PISP_PCMD(ccb)->wdog, ts, isp_watchdog, ccb);
4982                         break;
4983                 case CMD_RQLATER:
4984                         /*
4985                          * We get this result for FC devices if the loop state isn't ready yet
4986                          * or if the device in question has gone zombie on us.
4987                          *
4988                          * If we've never seen Loop UP at all, we requeue this request and wait
4989                          * for the initial loop up delay to expire.
4990                          */
4991                         lim = ISP_FC_PC(isp, bus)->loop_down_limit;
4992                         if (FCPARAM(isp, bus)->loop_seen_once == 0 || ISP_FC_PC(isp, bus)->loop_down_time >= lim) {
4993                                 if (FCPARAM(isp, bus)->loop_seen_once == 0) {
4994                                         isp_prt(isp, ISP_LOGDEBUG0, "%d.%d loop not seen yet @ %lu", XS_TGT(ccb), XS_LUN(ccb), (unsigned long) time_uptime);
4995                                 } else {
4996                                         isp_prt(isp, ISP_LOGDEBUG0, "%d.%d downtime (%d) > lim (%d)", XS_TGT(ccb), XS_LUN(ccb), ISP_FC_PC(isp, bus)->loop_down_time, lim);
4997                                 }
4998                                 ccb->ccb_h.status = CAM_SEL_TIMEOUT;
4999                                 isp_done((struct ccb_scsiio *) ccb);
5000                                 break;
5001                         }
5002                         isp_prt(isp, ISP_LOGDEBUG0, "%d.%d retry later", XS_TGT(ccb), XS_LUN(ccb));
5003                         cam_freeze_devq(ccb->ccb_h.path);
5004                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 1000, 0);
5005                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
5006                         isp_free_pcmd(isp, ccb);
5007                         xpt_done(ccb);
5008                         break;
5009                 case CMD_EAGAIN:
5010                         isp_free_pcmd(isp, ccb);
5011                         cam_freeze_devq(ccb->ccb_h.path);
5012                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 100, 0);
5013                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
5014                         xpt_done(ccb);
5015                         break;
5016                 case CMD_COMPLETE:
5017                         isp_done((struct ccb_scsiio *) ccb);
5018                         break;
5019                 default:
5020                         isp_prt(isp, ISP_LOGERR, "What's this? 0x%x at %d in file %s", error, __LINE__, __FILE__);
5021                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
5022                         isp_free_pcmd(isp, ccb);
5023                         xpt_done(ccb);
5024                 }
5025                 break;
5026
5027 #ifdef  ISP_TARGET_MODE
5028         case XPT_EN_LUN:                /* Enable/Disable LUN as a target */
5029                 if (ccb->cel.enable) {
5030                         isp_enable_lun(isp, ccb);
5031                 } else {
5032                         isp_disable_lun(isp, ccb);
5033                 }
5034                 break;
5035         case XPT_IMMED_NOTIFY:
5036         case XPT_IMMEDIATE_NOTIFY:      /* Add Immediate Notify Resource */
5037         case XPT_ACCEPT_TARGET_IO:      /* Add Accept Target IO Resource */
5038         {
5039                 tstate_t *tptr = get_lun_statep(isp, XS_CHANNEL(ccb), ccb->ccb_h.target_lun);
5040                 if (tptr == NULL) {
5041                         tptr = get_lun_statep(isp, XS_CHANNEL(ccb), CAM_LUN_WILDCARD);
5042                 }
5043                 if (tptr == NULL) {
5044                         const char *str;
5045                         uint32_t tag;
5046
5047                         if (ccb->ccb_h.func_code == XPT_IMMEDIATE_NOTIFY) {
5048                                 str = "XPT_IMMEDIATE_NOTIFY";
5049                                 tag = ccb->cin1.seq_id;
5050                         } else {
5051                                 tag = ccb->atio.tag_id;
5052                                 str = "XPT_ACCEPT_TARGET_IO";
5053                         }
5054                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "%s: [0x%x] no state pointer found for %s\n", __func__, tag, str);
5055                         dump_tstates(isp, XS_CHANNEL(ccb));
5056                         ccb->ccb_h.status = CAM_DEV_NOT_THERE;
5057                         break;
5058                 }
5059                 ccb->ccb_h.spriv_field0 = 0;
5060                 ccb->ccb_h.spriv_ptr1 = isp;
5061
5062                 if (ccb->ccb_h.func_code == XPT_ACCEPT_TARGET_IO) {
5063                         if (ccb->atio.tag_id) {
5064                                 atio_private_data_t *atp = isp_find_atpd(isp, tptr, ccb->atio.tag_id);
5065                                 if (atp) {
5066                                         isp_put_atpd(isp, tptr, atp);
5067                                 }
5068                         }
5069                         tptr->atio_count++;
5070                         SLIST_INSERT_HEAD(&tptr->atios, &ccb->ccb_h, sim_links.sle);
5071                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, ccb->ccb_h.path, "Put FREE ATIO (tag id 0x%x), count now %d\n",
5072                             ccb->atio.tag_id, tptr->atio_count);
5073                         ccb->atio.tag_id = 0;
5074                 } else if (ccb->ccb_h.func_code == XPT_IMMEDIATE_NOTIFY) {
5075                         if (ccb->cin1.tag_id) {
5076                                 inot_private_data_t *ntp = isp_find_ntpd(isp, tptr, ccb->cin1.tag_id, ccb->cin1.seq_id);
5077                                 if (ntp) {
5078                                         isp_put_ntpd(isp, tptr, ntp);
5079                                 }
5080                         }
5081                         tptr->inot_count++;
5082                         SLIST_INSERT_HEAD(&tptr->inots, &ccb->ccb_h, sim_links.sle);
5083                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, ccb->ccb_h.path, "Put FREE INOT, (seq id 0x%x) count now %d\n",
5084                             ccb->cin1.seq_id, tptr->inot_count);
5085                         ccb->cin1.seq_id = 0;
5086                 } else if (ccb->ccb_h.func_code == XPT_IMMED_NOTIFY) {
5087                         tptr->inot_count++;
5088                         SLIST_INSERT_HEAD(&tptr->inots, &ccb->ccb_h, sim_links.sle);
5089                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, ccb->ccb_h.path, "Put FREE INOT, (seq id 0x%x) count now %d\n",
5090                             ccb->cin1.seq_id, tptr->inot_count);
5091                         ccb->cin1.seq_id = 0;
5092                 }
5093                 rls_lun_statep(isp, tptr);
5094                 ccb->ccb_h.status = CAM_REQ_INPROG;
5095                 break;
5096         }
5097         case XPT_NOTIFY_ACK:
5098                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
5099                 break;
5100         case XPT_NOTIFY_ACKNOWLEDGE:            /* notify ack */
5101         {
5102                 tstate_t *tptr;
5103                 inot_private_data_t *ntp;
5104
5105                 /*
5106                  * XXX: Because we cannot guarantee that the path information in the notify acknowledge ccb
5107                  * XXX: matches that for the immediate notify, we have to *search* for the notify structure
5108                  */
5109                 /*
5110                  * All the relevant path information is in the associated immediate notify
5111                  */
5112                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: [0x%x] NOTIFY ACKNOWLEDGE for 0x%x seen\n", __func__, ccb->cna2.tag_id, ccb->cna2.seq_id);
5113                 ntp = get_ntp_from_tagdata(isp, ccb->cna2.tag_id, ccb->cna2.seq_id, &tptr);
5114                 if (ntp == NULL) {
5115                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "%s: [0x%x] XPT_NOTIFY_ACKNOWLEDGE of 0x%x cannot find ntp private data\n", __func__,
5116                              ccb->cna2.tag_id, ccb->cna2.seq_id);
5117                         ccb->ccb_h.status = CAM_DEV_NOT_THERE;
5118                         xpt_done(ccb);
5119                         break;
5120                 }
5121                 if (isp_handle_platform_target_notify_ack(isp, &ntp->rd.nt)) {
5122                         rls_lun_statep(isp, tptr);
5123                         cam_freeze_devq(ccb->ccb_h.path);
5124                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 1000, 0);
5125                         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
5126                         ccb->ccb_h.status |= CAM_REQUEUE_REQ;
5127                         break;
5128                 }
5129                 isp_put_ntpd(isp, tptr, ntp);
5130                 rls_lun_statep(isp, tptr);
5131                 ccb->ccb_h.status = CAM_REQ_CMP;
5132                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: [0x%x] calling xpt_done for tag 0x%x\n", __func__, ccb->cna2.tag_id, ccb->cna2.seq_id);
5133                 xpt_done(ccb);
5134                 break;
5135         }
5136         case XPT_CONT_TARGET_IO:
5137                 isp_target_start_ctio(isp, ccb, FROM_CAM);
5138                 break;
5139 #endif
5140         case XPT_RESET_DEV:             /* BDR the specified SCSI device */
5141         {
5142                 struct isp_fc *fc;
5143
5144                 bus = cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
5145                 tgt = ccb->ccb_h.target_id;
5146                 tgt |= (bus << 16);
5147                 if (IS_FC(isp))
5148                         fc = ISP_FC_PC(isp, bus);
5149                 else
5150                         fc = NULL;
5151
5152                 error = isp_control(isp, ISPCTL_RESET_DEV, bus, tgt);
5153                 if (error) {
5154                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
5155                 } else {
5156                         /*
5157                          * If we have a FC device, reset the Command
5158                          * Reference Number, because the target will expect
5159                          * that we re-start the CRN at 1 after a reset.
5160                          */
5161                         if (fc != NULL)
5162                                 isp_fcp_reset_crn(fc, tgt, /*tgt_set*/ 1);
5163
5164                         ccb->ccb_h.status = CAM_REQ_CMP;
5165                 }
5166                 xpt_done(ccb);
5167                 break;
5168         }
5169         case XPT_ABORT:                 /* Abort the specified CCB */
5170         {
5171                 union ccb *accb = ccb->cab.abort_ccb;
5172                 switch (accb->ccb_h.func_code) {
5173 #ifdef  ISP_TARGET_MODE
5174                 case XPT_ACCEPT_TARGET_IO:
5175                         isp_target_mark_aborted(isp, ccb);
5176                         break;
5177 #endif
5178                 case XPT_SCSI_IO:
5179                         error = isp_control(isp, ISPCTL_ABORT_CMD, accb);
5180                         if (error) {
5181                                 ccb->ccb_h.status = CAM_UA_ABORT;
5182                         } else {
5183                                 ccb->ccb_h.status = CAM_REQ_CMP;
5184                         }
5185                         break;
5186                 default:
5187                         ccb->ccb_h.status = CAM_REQ_INVALID;
5188                         break;
5189                 }
5190                 /*
5191                  * This is not a queued CCB, so the caller expects it to be
5192                  * complete when control is returned.
5193                  */
5194                 break;
5195         }
5196 #define IS_CURRENT_SETTINGS(c)  (c->type == CTS_TYPE_CURRENT_SETTINGS)
5197         case XPT_SET_TRAN_SETTINGS:     /* Nexus Settings */
5198                 cts = &ccb->cts;
5199                 if (!IS_CURRENT_SETTINGS(cts)) {
5200                         ccb->ccb_h.status = CAM_REQ_INVALID;
5201                         xpt_done(ccb);
5202                         break;
5203                 }
5204                 tgt = cts->ccb_h.target_id;
5205                 bus = cam_sim_bus(xpt_path_sim(cts->ccb_h.path));
5206                 if (IS_SCSI(isp)) {
5207                         struct ccb_trans_settings_scsi *scsi = &cts->proto_specific.scsi;
5208                         struct ccb_trans_settings_spi *spi = &cts->xport_specific.spi;
5209                         sdparam *sdp = SDPARAM(isp, bus);
5210                         uint16_t *dptr;
5211
5212                         if (spi->valid == 0 && scsi->valid == 0) {
5213                                 ccb->ccb_h.status = CAM_REQ_CMP;
5214                                 xpt_done(ccb);
5215                                 break;
5216                         }
5217
5218                         /*
5219                          * We always update (internally) from goal_flags
5220                          * so any request to change settings just gets
5221                          * vectored to that location.
5222                          */
5223                         dptr = &sdp->isp_devparam[tgt].goal_flags;
5224
5225                         if ((spi->valid & CTS_SPI_VALID_DISC) != 0) {
5226                                 if ((spi->flags & CTS_SPI_FLAGS_DISC_ENB) != 0)
5227                                         *dptr |= DPARM_DISC;
5228                                 else
5229                                         *dptr &= ~DPARM_DISC;
5230                         }
5231
5232                         if ((scsi->valid & CTS_SCSI_VALID_TQ) != 0) {
5233                                 if ((scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0)
5234                                         *dptr |= DPARM_TQING;
5235                                 else
5236                                         *dptr &= ~DPARM_TQING;
5237                         }
5238
5239                         if ((spi->valid & CTS_SPI_VALID_BUS_WIDTH) != 0) {
5240                                 if (spi->bus_width == MSG_EXT_WDTR_BUS_16_BIT)
5241                                         *dptr |= DPARM_WIDE;
5242                                 else
5243                                         *dptr &= ~DPARM_WIDE;
5244                         }
5245
5246                         /*
5247                          * XXX: FIX ME
5248                          */
5249                         if ((spi->valid & CTS_SPI_VALID_SYNC_OFFSET) && (spi->valid & CTS_SPI_VALID_SYNC_RATE) && (spi->sync_period && spi->sync_offset)) {
5250                                 *dptr |= DPARM_SYNC;
5251                                 /*
5252                                  * XXX: CHECK FOR LEGALITY
5253                                  */
5254                                 sdp->isp_devparam[tgt].goal_period = spi->sync_period;
5255                                 sdp->isp_devparam[tgt].goal_offset = spi->sync_offset;
5256                         } else {
5257                                 *dptr &= ~DPARM_SYNC;
5258                         }
5259                         isp_prt(isp, ISP_LOGDEBUG0, "SET (%d.%d.%d) to flags %x off %x per %x", bus, tgt, cts->ccb_h.target_lun, sdp->isp_devparam[tgt].goal_flags,
5260                             sdp->isp_devparam[tgt].goal_offset, sdp->isp_devparam[tgt].goal_period);
5261                         sdp->isp_devparam[tgt].dev_update = 1;
5262                         sdp->update = 1;
5263                 }
5264                 ccb->ccb_h.status = CAM_REQ_CMP;
5265                 xpt_done(ccb);
5266                 break;
5267         case XPT_GET_TRAN_SETTINGS:
5268                 cts = &ccb->cts;
5269                 tgt = cts->ccb_h.target_id;
5270                 bus = cam_sim_bus(xpt_path_sim(cts->ccb_h.path));
5271                 if (IS_FC(isp)) {
5272                         fcparam *fcp = FCPARAM(isp, bus);
5273                         struct ccb_trans_settings_scsi *scsi = &cts->proto_specific.scsi;
5274                         struct ccb_trans_settings_fc *fc = &cts->xport_specific.fc;
5275                         unsigned int hdlidx;
5276
5277                         cts->protocol = PROTO_SCSI;
5278                         cts->protocol_version = SCSI_REV_2;
5279                         cts->transport = XPORT_FC;
5280                         cts->transport_version = 0;
5281
5282                         scsi->valid = CTS_SCSI_VALID_TQ;
5283                         scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
5284                         fc->valid = CTS_FC_VALID_SPEED;
5285                         fc->bitrate = 100000;
5286                         fc->bitrate *= fcp->isp_gbspeed;
5287                         hdlidx = fcp->isp_dev_map[tgt] - 1;
5288                         if (hdlidx < MAX_FC_TARG) {
5289                                 fcportdb_t *lp = &fcp->portdb[hdlidx];
5290                                 fc->wwnn = lp->node_wwn;
5291                                 fc->wwpn = lp->port_wwn;
5292                                 fc->port = lp->portid;
5293                                 fc->valid |= CTS_FC_VALID_WWNN | CTS_FC_VALID_WWPN | CTS_FC_VALID_PORT;
5294                         }
5295                 } else {
5296                         struct ccb_trans_settings_scsi *scsi = &cts->proto_specific.scsi;
5297                         struct ccb_trans_settings_spi *spi = &cts->xport_specific.spi;
5298                         sdparam *sdp = SDPARAM(isp, bus);
5299                         uint16_t dval, pval, oval;
5300
5301                         if (IS_CURRENT_SETTINGS(cts)) {
5302                                 sdp->isp_devparam[tgt].dev_refresh = 1;
5303                                 sdp->update = 1;
5304                                 (void) isp_control(isp, ISPCTL_UPDATE_PARAMS, bus);
5305                                 dval = sdp->isp_devparam[tgt].actv_flags;
5306                                 oval = sdp->isp_devparam[tgt].actv_offset;
5307                                 pval = sdp->isp_devparam[tgt].actv_period;
5308                         } else {
5309                                 dval = sdp->isp_devparam[tgt].nvrm_flags;
5310                                 oval = sdp->isp_devparam[tgt].nvrm_offset;
5311                                 pval = sdp->isp_devparam[tgt].nvrm_period;
5312                         }
5313
5314                         cts->protocol = PROTO_SCSI;
5315                         cts->protocol_version = SCSI_REV_2;
5316                         cts->transport = XPORT_SPI;
5317                         cts->transport_version = 2;
5318
5319                         spi->valid = 0;
5320                         scsi->valid = 0;
5321                         spi->flags = 0;
5322                         scsi->flags = 0;
5323                         if (dval & DPARM_DISC) {
5324                                 spi->flags |= CTS_SPI_FLAGS_DISC_ENB;
5325                         }
5326                         if ((dval & DPARM_SYNC) && oval && pval) {
5327                                 spi->sync_offset = oval;
5328                                 spi->sync_period = pval;
5329                         } else {
5330                                 spi->sync_offset = 0;
5331                                 spi->sync_period = 0;
5332                         }
5333                         spi->valid |= CTS_SPI_VALID_SYNC_OFFSET;
5334                         spi->valid |= CTS_SPI_VALID_SYNC_RATE;
5335                         spi->valid |= CTS_SPI_VALID_BUS_WIDTH;
5336                         if (dval & DPARM_WIDE) {
5337                                 spi->bus_width = MSG_EXT_WDTR_BUS_16_BIT;
5338                         } else {
5339                                 spi->bus_width = MSG_EXT_WDTR_BUS_8_BIT;
5340                         }
5341                         if (cts->ccb_h.target_lun != CAM_LUN_WILDCARD) {
5342                                 scsi->valid = CTS_SCSI_VALID_TQ;
5343                                 if (dval & DPARM_TQING) {
5344                                         scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
5345                                 }
5346                                 spi->valid |= CTS_SPI_VALID_DISC;
5347                         }
5348                         isp_prt(isp, ISP_LOGDEBUG0, "GET %s (%d.%d.%d) to flags %x off %x per %x", IS_CURRENT_SETTINGS(cts)? "ACTIVE" : "NVRAM",
5349                             bus, tgt, cts->ccb_h.target_lun, dval, oval, pval);
5350                 }
5351                 ccb->ccb_h.status = CAM_REQ_CMP;
5352                 xpt_done(ccb);
5353                 break;
5354
5355         case XPT_CALC_GEOMETRY:
5356                 cam_calc_geometry(&ccb->ccg, 1);
5357                 xpt_done(ccb);
5358                 break;
5359
5360         case XPT_RESET_BUS:             /* Reset the specified bus */
5361                 bus = cam_sim_bus(sim);
5362                 error = isp_control(isp, ISPCTL_RESET_BUS, bus);
5363                 if (error) {
5364                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
5365                         xpt_done(ccb);
5366                         break;
5367                 }
5368                 if (bootverbose) {
5369                         xpt_print(ccb->ccb_h.path, "reset bus on channel %d\n", bus);
5370                 }
5371                 if (IS_FC(isp)) {
5372                         xpt_async(AC_BUS_RESET, ISP_FC_PC(isp, bus)->path, 0);
5373                 } else {
5374                         xpt_async(AC_BUS_RESET, ISP_SPI_PC(isp, bus)->path, 0);
5375                 }
5376                 ccb->ccb_h.status = CAM_REQ_CMP;
5377                 xpt_done(ccb);
5378                 break;
5379
5380         case XPT_TERM_IO:               /* Terminate the I/O process */
5381                 ccb->ccb_h.status = CAM_REQ_INVALID;
5382                 xpt_done(ccb);
5383                 break;
5384
5385         case XPT_SET_SIM_KNOB:          /* Set SIM knobs */
5386         {
5387                 struct ccb_sim_knob *kp = &ccb->knob;
5388                 fcparam *fcp;
5389
5390                 if (!IS_FC(isp)) {
5391                         ccb->ccb_h.status = CAM_REQ_INVALID;
5392                         xpt_done(ccb);
5393                         break;
5394                 }
5395
5396                 bus = cam_sim_bus(xpt_path_sim(kp->ccb_h.path));
5397                 fcp = FCPARAM(isp, bus);
5398
5399                 if (kp->xport_specific.fc.valid & KNOB_VALID_ADDRESS) {
5400                         fcp->isp_wwnn = ISP_FC_PC(isp, bus)->def_wwnn = kp->xport_specific.fc.wwnn;
5401                         fcp->isp_wwpn = ISP_FC_PC(isp, bus)->def_wwpn = kp->xport_specific.fc.wwpn;
5402                         isp_prt(isp, ISP_LOGALL, "Setting Channel %d wwns to 0x%jx 0x%jx", bus, fcp->isp_wwnn, fcp->isp_wwpn);
5403                 }
5404                 ccb->ccb_h.status = CAM_REQ_CMP;
5405                 if (kp->xport_specific.fc.valid & KNOB_VALID_ROLE) {
5406                         int rchange = 0;
5407                         int newrole = 0;
5408
5409                         switch (kp->xport_specific.fc.role) {
5410                         case KNOB_ROLE_NONE:
5411                                 if (fcp->role != ISP_ROLE_NONE) {
5412                                         rchange = 1;
5413                                         newrole = ISP_ROLE_NONE;
5414                                 }
5415                                 break;
5416                         case KNOB_ROLE_TARGET:
5417                                 if (fcp->role != ISP_ROLE_TARGET) {
5418                                         rchange = 1;
5419                                         newrole = ISP_ROLE_TARGET;
5420                                 }
5421                                 break;
5422                         case KNOB_ROLE_INITIATOR:
5423                                 if (fcp->role != ISP_ROLE_INITIATOR) {
5424                                         rchange = 1;
5425                                         newrole = ISP_ROLE_INITIATOR;
5426                                 }
5427                                 break;
5428                         case KNOB_ROLE_BOTH:
5429 #if 0
5430                                 if (fcp->role != ISP_ROLE_BOTH) {
5431                                         rchange = 1;
5432                                         newrole = ISP_ROLE_BOTH;
5433                                 }
5434 #else
5435                                 /*
5436                                  * We don't really support dual role at present on FC cards.
5437                                  *
5438                                  * We should, but a bunch of things are currently broken,
5439                                  * so don't allow it.
5440                                  */
5441                                 isp_prt(isp, ISP_LOGERR, "cannot support dual role at present");
5442                                 ccb->ccb_h.status = CAM_REQ_INVALID;
5443 #endif
5444                                 break;
5445                         }
5446                         if (rchange) {
5447                                 ISP_PATH_PRT(isp, ISP_LOGCONFIG, ccb->ccb_h.path, "changing role on from %d to %d\n", fcp->role, newrole);
5448 #ifdef  ISP_TARGET_MODE
5449                                 ISP_SET_PC(isp, bus, tm_enabled, 0);
5450                                 ISP_SET_PC(isp, bus, tm_luns_enabled, 0);
5451 #endif
5452                                 if (isp_fc_change_role(isp, bus, newrole) != 0) {
5453                                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
5454                                         xpt_done(ccb);
5455                                         break;
5456                                 }
5457 #ifdef  ISP_TARGET_MODE
5458                                 if (newrole == ISP_ROLE_TARGET || newrole == ISP_ROLE_BOTH) {
5459                                         /*
5460                                          * Give the new role a chance to complain and settle
5461                                          */
5462                                         msleep(isp, &isp->isp_lock, PRIBIO, "taking a breather", 2);
5463                                         ccb->ccb_h.status = isp_enable_deferred_luns(isp, bus);
5464                                 }
5465 #endif
5466                         }
5467                 }
5468                 xpt_done(ccb);
5469                 break;
5470         }
5471         case XPT_GET_SIM_KNOB:          /* Get SIM knobs */
5472         {
5473                 struct ccb_sim_knob *kp = &ccb->knob;
5474
5475                 if (IS_FC(isp)) {
5476                         fcparam *fcp;
5477
5478                         bus = cam_sim_bus(xpt_path_sim(kp->ccb_h.path));
5479                         fcp = FCPARAM(isp, bus);
5480
5481                         kp->xport_specific.fc.wwnn = fcp->isp_wwnn;
5482                         kp->xport_specific.fc.wwpn = fcp->isp_wwpn;
5483                         switch (fcp->role) {
5484                         case ISP_ROLE_NONE:
5485                                 kp->xport_specific.fc.role = KNOB_ROLE_NONE;
5486                                 break;
5487                         case ISP_ROLE_TARGET:
5488                                 kp->xport_specific.fc.role = KNOB_ROLE_TARGET;
5489                                 break;
5490                         case ISP_ROLE_INITIATOR:
5491                                 kp->xport_specific.fc.role = KNOB_ROLE_INITIATOR;
5492                                 break;
5493                         case ISP_ROLE_BOTH:
5494                                 kp->xport_specific.fc.role = KNOB_ROLE_BOTH;
5495                                 break;
5496                         }
5497                         kp->xport_specific.fc.valid = KNOB_VALID_ADDRESS | KNOB_VALID_ROLE;
5498                         ccb->ccb_h.status = CAM_REQ_CMP;
5499                 } else {
5500                         ccb->ccb_h.status = CAM_REQ_INVALID;
5501                 }
5502                 xpt_done(ccb);
5503                 break;
5504         }
5505         case XPT_PATH_INQ:              /* Path routing inquiry */
5506         {
5507                 struct ccb_pathinq *cpi = &ccb->cpi;
5508
5509                 cpi->version_num = 1;
5510 #ifdef  ISP_TARGET_MODE
5511                 cpi->target_sprt = PIT_PROCESSOR | PIT_DISCONNECT | PIT_TERM_IO;
5512 #else
5513                 cpi->target_sprt = 0;
5514 #endif
5515                 cpi->hba_eng_cnt = 0;
5516                 cpi->max_target = ISP_MAX_TARGETS(isp) - 1;
5517                 cpi->max_lun = ISP_MAX_LUNS(isp) - 1;
5518                 cpi->bus_id = cam_sim_bus(sim);
5519                 if (isp->isp_osinfo.sixtyfourbit)
5520                         cpi->maxio = (ISP_NSEG64_MAX - 1) * PAGE_SIZE;
5521                 else
5522                         cpi->maxio = (ISP_NSEG_MAX - 1) * PAGE_SIZE;
5523
5524                 bus = cam_sim_bus(xpt_path_sim(cpi->ccb_h.path));
5525                 if (IS_FC(isp)) {
5526                         fcparam *fcp = FCPARAM(isp, bus);
5527
5528                         cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
5529
5530                         /*
5531                          * Because our loop ID can shift from time to time,
5532                          * make our initiator ID out of range of our bus.
5533                          */
5534                         cpi->initiator_id = cpi->max_target + 1;
5535
5536                         /*
5537                          * Set base transfer capabilities for Fibre Channel, for this HBA.
5538                          */
5539                         if (IS_25XX(isp)) {
5540                                 cpi->base_transfer_speed = 8000000;
5541                         } else if (IS_24XX(isp)) {
5542                                 cpi->base_transfer_speed = 4000000;
5543                         } else if (IS_23XX(isp)) {
5544                                 cpi->base_transfer_speed = 2000000;
5545                         } else {
5546                                 cpi->base_transfer_speed = 1000000;
5547                         }
5548                         cpi->hba_inquiry = PI_TAG_ABLE;
5549                         cpi->transport = XPORT_FC;
5550                         cpi->transport_version = 0;
5551                         cpi->xport_specific.fc.wwnn = fcp->isp_wwnn;
5552                         cpi->xport_specific.fc.wwpn = fcp->isp_wwpn;
5553                         cpi->xport_specific.fc.port = fcp->isp_portid;
5554                         cpi->xport_specific.fc.bitrate = fcp->isp_gbspeed * 1000;
5555                 } else {
5556                         sdparam *sdp = SDPARAM(isp, bus);
5557                         cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
5558                         cpi->hba_misc = PIM_UNMAPPED;
5559                         cpi->initiator_id = sdp->isp_initiator_id;
5560                         cpi->base_transfer_speed = 3300;
5561                         cpi->transport = XPORT_SPI;
5562                         cpi->transport_version = 2;
5563                 }
5564                 cpi->protocol = PROTO_SCSI;
5565                 cpi->protocol_version = SCSI_REV_2;
5566                 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
5567                 strncpy(cpi->hba_vid, "Qlogic", HBA_IDLEN);
5568                 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
5569                 cpi->unit_number = cam_sim_unit(sim);
5570                 cpi->ccb_h.status = CAM_REQ_CMP;
5571                 xpt_done(ccb);
5572                 break;
5573         }
5574         default:
5575                 ccb->ccb_h.status = CAM_REQ_INVALID;
5576                 xpt_done(ccb);
5577                 break;
5578         }
5579 }
5580
5581 #define ISPDDB  (CAM_DEBUG_INFO|CAM_DEBUG_TRACE|CAM_DEBUG_CDB)
5582
5583 void
5584 isp_done(XS_T *sccb)
5585 {
5586         ispsoftc_t *isp = XS_ISP(sccb);
5587         uint32_t status;
5588
5589         if (XS_NOERR(sccb))
5590                 XS_SETERR(sccb, CAM_REQ_CMP);
5591
5592         if ((sccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP && (sccb->scsi_status != SCSI_STATUS_OK)) {
5593                 sccb->ccb_h.status &= ~CAM_STATUS_MASK;
5594                 if ((sccb->scsi_status == SCSI_STATUS_CHECK_COND) && (sccb->ccb_h.status & CAM_AUTOSNS_VALID) == 0) {
5595                         sccb->ccb_h.status |= CAM_AUTOSENSE_FAIL;
5596                 } else {
5597                         sccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
5598                 }
5599         }
5600
5601         sccb->ccb_h.status &= ~CAM_SIM_QUEUED;
5602         status = sccb->ccb_h.status & CAM_STATUS_MASK;
5603         if (status != CAM_REQ_CMP) {
5604                 if (status != CAM_SEL_TIMEOUT)
5605                         isp_prt(isp, ISP_LOGDEBUG0, "target %d lun %d CAM status 0x%x SCSI status 0x%x", XS_TGT(sccb), XS_LUN(sccb), sccb->ccb_h.status, sccb->scsi_status);
5606                 else if ((IS_FC(isp))
5607                       && (XS_TGT(sccb) < MAX_FC_TARG)) {
5608                         fcparam *fcp;
5609                         int hdlidx;
5610
5611                         fcp = FCPARAM(isp, XS_CHANNEL(sccb));
5612                         hdlidx = fcp->isp_dev_map[XS_TGT(sccb)] - 1;
5613                         /*
5614                          * Note that we have reported that this device is
5615                          * gone.  If it reappears, we'll need to issue a
5616                          * rescan.
5617                          */
5618                         if (hdlidx >= 0 && hdlidx < MAX_FC_TARG)
5619                                 fcp->portdb[hdlidx].reported_gone = 1;
5620                 }
5621                 if ((sccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
5622                         sccb->ccb_h.status |= CAM_DEV_QFRZN;
5623                         xpt_freeze_devq(sccb->ccb_h.path, 1);
5624                 }
5625         }
5626
5627         if ((CAM_DEBUGGED(sccb->ccb_h.path, ISPDDB)) && (sccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
5628                 xpt_print(sccb->ccb_h.path, "cam completion status 0x%x\n", sccb->ccb_h.status);
5629         }
5630
5631         if (ISP_PCMD(sccb)) {
5632                 if (callout_active(&PISP_PCMD(sccb)->wdog))
5633                         callout_stop(&PISP_PCMD(sccb)->wdog);
5634                 isp_free_pcmd(isp, (union ccb *) sccb);
5635         }
5636         xpt_done((union ccb *) sccb);
5637 }
5638
5639 void
5640 isp_async(ispsoftc_t *isp, ispasync_t cmd, ...)
5641 {
5642         int bus;
5643         static const char prom0[] = "Chan %d PortID 0x%06x handle 0x%x %s %s WWPN 0x%08x%08x";
5644         static const char prom2[] = "Chan %d PortID 0x%06x handle 0x%x %s %s tgt %u WWPN 0x%08x%08x";
5645         char buf[64];
5646         char *msg = NULL;
5647         target_id_t tgt;
5648         fcportdb_t *lp;
5649         struct isp_fc *fc;
5650         struct cam_path *tmppath;
5651         va_list ap;
5652
5653         switch (cmd) {
5654         case ISPASYNC_NEW_TGT_PARAMS:
5655         {
5656                 struct ccb_trans_settings_scsi *scsi;
5657                 struct ccb_trans_settings_spi *spi;
5658                 int flags, tgt;
5659                 sdparam *sdp;
5660                 struct ccb_trans_settings cts;
5661
5662                 memset(&cts, 0, sizeof (struct ccb_trans_settings));
5663
5664                 va_start(ap, cmd);
5665                 bus = va_arg(ap, int);
5666                 tgt = va_arg(ap, int);
5667                 va_end(ap);
5668                 sdp = SDPARAM(isp, bus);
5669
5670                 if (xpt_create_path(&tmppath, NULL, cam_sim_path(ISP_SPI_PC(isp, bus)->sim), tgt, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
5671                         isp_prt(isp, ISP_LOGWARN, "isp_async cannot make temp path for %d.%d", tgt, bus);
5672                         break;
5673                 }
5674                 flags = sdp->isp_devparam[tgt].actv_flags;
5675                 cts.type = CTS_TYPE_CURRENT_SETTINGS;
5676                 cts.protocol = PROTO_SCSI;
5677                 cts.transport = XPORT_SPI;
5678
5679                 scsi = &cts.proto_specific.scsi;
5680                 spi = &cts.xport_specific.spi;
5681
5682                 if (flags & DPARM_TQING) {
5683                         scsi->valid |= CTS_SCSI_VALID_TQ;
5684                         scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
5685                 }
5686
5687                 if (flags & DPARM_DISC) {
5688                         spi->valid |= CTS_SPI_VALID_DISC;
5689                         spi->flags |= CTS_SPI_FLAGS_DISC_ENB;
5690                 }
5691                 spi->flags |= CTS_SPI_VALID_BUS_WIDTH;
5692                 if (flags & DPARM_WIDE) {
5693                         spi->bus_width = MSG_EXT_WDTR_BUS_16_BIT;
5694                 } else {
5695                         spi->bus_width = MSG_EXT_WDTR_BUS_8_BIT;
5696                 }
5697                 if (flags & DPARM_SYNC) {
5698                         spi->valid |= CTS_SPI_VALID_SYNC_RATE;
5699                         spi->valid |= CTS_SPI_VALID_SYNC_OFFSET;
5700                         spi->sync_period = sdp->isp_devparam[tgt].actv_period;
5701                         spi->sync_offset = sdp->isp_devparam[tgt].actv_offset;
5702                 }
5703                 isp_prt(isp, ISP_LOGDEBUG2, "NEW_TGT_PARAMS bus %d tgt %d period %x offset %x flags %x", bus, tgt, sdp->isp_devparam[tgt].actv_period, sdp->isp_devparam[tgt].actv_offset, flags);
5704                 xpt_setup_ccb(&cts.ccb_h, tmppath, 1);
5705                 xpt_async(AC_TRANSFER_NEG, tmppath, &cts);
5706                 xpt_free_path(tmppath);
5707                 break;
5708         }
5709         case ISPASYNC_BUS_RESET:
5710         {
5711                 va_start(ap, cmd);
5712                 bus = va_arg(ap, int);
5713                 va_end(ap);
5714                 isp_prt(isp, ISP_LOGINFO, "SCSI bus reset on bus %d detected", bus);
5715                 if (IS_FC(isp)) {
5716                         xpt_async(AC_BUS_RESET, ISP_FC_PC(isp, bus)->path, NULL);
5717                 } else {
5718                         xpt_async(AC_BUS_RESET, ISP_SPI_PC(isp, bus)->path, NULL);
5719                 }
5720                 break;
5721         }
5722         case ISPASYNC_LIP:
5723                 if (msg == NULL) {
5724                         msg = "LIP Received";
5725                 }
5726                 /* FALLTHROUGH */
5727         case ISPASYNC_LOOP_RESET:
5728                 if (msg == NULL) {
5729                         msg = "LOOP Reset";
5730                 }
5731                 /* FALLTHROUGH */
5732         case ISPASYNC_LOOP_DOWN:
5733         {
5734                 if (msg == NULL) {
5735                         msg = "LOOP Down";
5736                 }
5737                 va_start(ap, cmd);
5738                 bus = va_arg(ap, int);
5739                 va_end(ap);
5740
5741                 FCPARAM(isp, bus)->link_active = 0;
5742
5743                 fc = ISP_FC_PC(isp, bus);
5744                 if (cmd == ISPASYNC_LOOP_DOWN && fc->ready) {
5745                         /*
5746                          * We don't do any simq freezing if we are only in target mode
5747                          */
5748                         if (FCPARAM(isp, bus)->role & ISP_ROLE_INITIATOR) {
5749                                 if (fc->path) {
5750                                         isp_freeze_loopdown(isp, bus, msg);
5751                                 }
5752                                 if (!callout_active(&fc->ldt)) {
5753                                         callout_reset(&fc->ldt, fc->loop_down_limit * hz, isp_ldt, fc);
5754                                         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Starting Loop Down Timer @ %lu", (unsigned long) time_uptime);
5755                                 }
5756                         }
5757                 }
5758                 isp_fcp_reset_crn(fc, /*tgt*/0, /*tgt_set*/ 0);
5759
5760                 isp_prt(isp, ISP_LOGINFO, "Chan %d: %s", bus, msg);
5761                 break;
5762         }
5763         case ISPASYNC_LOOP_UP:
5764                 va_start(ap, cmd);
5765                 bus = va_arg(ap, int);
5766                 va_end(ap);
5767                 fc = ISP_FC_PC(isp, bus);
5768                 /*
5769                  * Now we just note that Loop has come up. We don't
5770                  * actually do anything because we're waiting for a
5771                  * Change Notify before activating the FC cleanup
5772                  * thread to look at the state of the loop again.
5773                  */
5774                 FCPARAM(isp, bus)->link_active = 1;
5775                 fc->loop_dead = 0;
5776                 fc->loop_down_time = 0;
5777                 isp_prt(isp, ISP_LOGINFO, "Chan %d Loop UP", bus);
5778                 break;
5779         case ISPASYNC_DEV_ARRIVED:
5780                 va_start(ap, cmd);
5781                 bus = va_arg(ap, int);
5782                 lp = va_arg(ap, fcportdb_t *);
5783                 va_end(ap);
5784                 fc = ISP_FC_PC(isp, bus);
5785                 lp->announced = 0;
5786                 lp->gone_timer = 0;
5787                 if ((FCPARAM(isp, bus)->role & ISP_ROLE_INITIATOR) && (lp->prli_word3 & PRLI_WD3_TARGET_FUNCTION)) {
5788                         int dbidx = lp - FCPARAM(isp, bus)->portdb;
5789                         int i;
5790
5791                         for (i = 0; i < MAX_FC_TARG; i++) {
5792                                 if (i >= FL_ID && i <= SNS_ID) {
5793                                         continue;
5794                                 }
5795                                 if (FCPARAM(isp, bus)->isp_dev_map[i] == 0) {
5796                                         break;
5797                                 }
5798                         }
5799                         if (i < MAX_FC_TARG) {
5800                                 FCPARAM(isp, bus)->isp_dev_map[i] = dbidx + 1;
5801                                 lp->dev_map_idx = i + 1;
5802                         } else {
5803                                 isp_prt(isp, ISP_LOGWARN, "out of target ids");
5804                                 isp_dump_portdb(isp, bus);
5805                         }
5806                 }
5807                 isp_gen_role_str(buf, sizeof (buf), lp->prli_word3);
5808                 if (lp->dev_map_idx) {
5809                         tgt = lp->dev_map_idx - 1;
5810                         isp_prt(isp, ISP_LOGCONFIG, prom2, bus, lp->portid, lp->handle, buf, "arrived at", tgt, (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5811                         isp_fcp_reset_crn(fc, tgt, /*tgt_set*/ 1);
5812                         isp_make_here(isp, lp, bus, tgt);
5813                 } else {
5814                         isp_prt(isp, ISP_LOGCONFIG, prom0, bus, lp->portid, lp->handle, buf, "arrived", (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5815                 }
5816                 break;
5817         case ISPASYNC_DEV_CHANGED:
5818                 va_start(ap, cmd);
5819                 bus = va_arg(ap, int);
5820                 lp = va_arg(ap, fcportdb_t *);
5821                 va_end(ap);
5822                 fc = ISP_FC_PC(isp, bus);
5823                 lp->announced = 0;
5824                 lp->gone_timer = 0;
5825                 if (isp_change_is_bad) {
5826                         lp->state = FC_PORTDB_STATE_NIL;
5827                         if (lp->dev_map_idx) {
5828                                 tgt = lp->dev_map_idx - 1;
5829                                 FCPARAM(isp, bus)->isp_dev_map[tgt] = 0;
5830                                 lp->dev_map_idx = 0;
5831                                 isp_prt(isp, ISP_LOGCONFIG, prom3, bus, lp->portid, tgt, "change is bad");
5832                                 isp_make_gone(isp, lp, bus, tgt);
5833                         } else {
5834                                 isp_gen_role_str(buf, sizeof (buf), lp->prli_word3);
5835                                 isp_prt(isp, ISP_LOGCONFIG, prom0, bus, lp->portid, lp->handle, buf, "changed and departed",
5836                                     (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5837                         }
5838                 } else {
5839                         lp->portid = lp->new_portid;
5840                         lp->prli_word3 = lp->new_prli_word3;
5841                         isp_gen_role_str(buf, sizeof (buf), lp->prli_word3);
5842                         if (lp->dev_map_idx) {
5843                                 int t = lp->dev_map_idx - 1;
5844                                 FCPARAM(isp, bus)->isp_dev_map[t] = (lp - FCPARAM(isp, bus)->portdb) + 1;
5845                                 tgt = lp->dev_map_idx - 1;
5846                                 isp_prt(isp, ISP_LOGCONFIG, prom2, bus, lp->portid, lp->handle, buf, "changed at", tgt,
5847                                     (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5848                                 isp_fcp_reset_crn(fc, tgt, /*tgt_set*/ 1);
5849                         } else {
5850                                 isp_prt(isp, ISP_LOGCONFIG, prom0, bus, lp->portid, lp->handle, buf, "changed", (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5851                         }
5852                 }
5853                 break;
5854         case ISPASYNC_DEV_STAYED:
5855                 va_start(ap, cmd);
5856                 bus = va_arg(ap, int);
5857                 lp = va_arg(ap, fcportdb_t *);
5858                 va_end(ap);
5859                 isp_gen_role_str(buf, sizeof (buf), lp->prli_word3);
5860                 if (lp->dev_map_idx) {
5861                         fc = ISP_FC_PC(isp, bus);
5862                         tgt = lp->dev_map_idx - 1;
5863                         isp_prt(isp, ISP_LOGCONFIG, prom2, bus, lp->portid, lp->handle, buf, "stayed at", tgt,
5864                             (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5865                         /*
5866                          * Only issue a rescan if we've actually reported
5867                          * that this device is gone.
5868                          */
5869                         if (lp->reported_gone != 0) {
5870                                 isp_prt(isp, ISP_LOGCONFIG, prom2, bus, lp->portid, lp->handle, buf, "rescanned at", tgt,
5871                                     (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5872                                 isp_make_here(isp, lp, bus, tgt);
5873                         }
5874                 } else {
5875                         isp_prt(isp, ISP_LOGCONFIG, prom0, bus, lp->portid, lp->handle, buf, "stayed",
5876                             (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5877                 }
5878                 break;
5879         case ISPASYNC_DEV_GONE:
5880                 va_start(ap, cmd);
5881                 bus = va_arg(ap, int);
5882                 lp = va_arg(ap, fcportdb_t *);
5883                 va_end(ap);
5884                 fc = ISP_FC_PC(isp, bus);
5885                 /*
5886                  * If this has a virtual target and we haven't marked it
5887                  * that we're going to have isp_gdt tell the OS it's gone,
5888                  * set the isp_gdt timer running on it.
5889                  *
5890                  * If it isn't marked that isp_gdt is going to get rid of it,
5891                  * announce that it's gone.
5892                  *
5893                  */
5894                 isp_gen_role_str(buf, sizeof (buf), lp->prli_word3);
5895                 if (lp->dev_map_idx && lp->announced == 0) {
5896                         lp->announced = 1;
5897                         lp->state = FC_PORTDB_STATE_ZOMBIE;
5898                         lp->gone_timer = ISP_FC_PC(isp, bus)->gone_device_time;
5899                         if (fc->ready && !callout_active(&fc->gdt)) {
5900                                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Starting Gone Device Timer with %u seconds time now %lu", bus, lp->gone_timer, (unsigned long)time_uptime);
5901                                 callout_reset(&fc->gdt, hz, isp_gdt, fc);
5902                         }
5903                         tgt = lp->dev_map_idx - 1;
5904                         isp_prt(isp, ISP_LOGCONFIG, prom2, bus, lp->portid, lp->handle, buf, "gone zombie at", tgt, (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5905                         isp_fcp_reset_crn(fc, tgt, /*tgt_set*/ 1);
5906                 } else if (lp->announced == 0) {
5907                         isp_prt(isp, ISP_LOGCONFIG, prom0, bus, lp->portid, lp->handle, buf, "departed", (uint32_t) (lp->port_wwn >> 32), (uint32_t) lp->port_wwn);
5908                 }
5909                 break;
5910         case ISPASYNC_CHANGE_NOTIFY:
5911         {
5912                 char *msg;
5913                 int evt, nphdl, nlstate, reason;
5914
5915                 va_start(ap, cmd);
5916                 bus = va_arg(ap, int);
5917                 evt = va_arg(ap, int);
5918                 if (IS_24XX(isp) && evt == ISPASYNC_CHANGE_PDB) {
5919                         nphdl = va_arg(ap, int);
5920                         nlstate = va_arg(ap, int);
5921                         reason = va_arg(ap, int);
5922                 } else {
5923                         nphdl = NIL_HANDLE;
5924                         nlstate = reason = 0;
5925                 }
5926                 va_end(ap);
5927                 fc = ISP_FC_PC(isp, bus);
5928
5929                 if (evt == ISPASYNC_CHANGE_PDB) {
5930                         msg = "Chan %d Port Database Changed";
5931                 } else if (evt == ISPASYNC_CHANGE_SNS) {
5932                         msg = "Chan %d Name Server Database Changed";
5933                 } else {
5934                         msg = "Chan %d Other Change Notify";
5935                 }
5936
5937                 /*
5938                  * If the loop down timer is running, cancel it.
5939                  */
5940                 if (fc->ready && callout_active(&fc->ldt)) {
5941                         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Stopping Loop Down Timer @ %lu", (unsigned long) time_uptime);
5942                         callout_stop(&fc->ldt);
5943                 }
5944                 isp_prt(isp, ISP_LOGINFO, msg, bus);
5945                 if (FCPARAM(isp, bus)->role & ISP_ROLE_INITIATOR) {
5946                         isp_freeze_loopdown(isp, bus, msg);
5947                 }
5948                 wakeup(fc);
5949                 break;
5950         }
5951 #ifdef  ISP_TARGET_MODE
5952         case ISPASYNC_TARGET_NOTIFY:
5953         {
5954                 isp_notify_t *notify;
5955                 va_start(ap, cmd);
5956                 notify = va_arg(ap, isp_notify_t *);
5957                 va_end(ap);
5958                 switch (notify->nt_ncode) {
5959                 case NT_ABORT_TASK:
5960                 case NT_ABORT_TASK_SET:
5961                 case NT_CLEAR_ACA:
5962                 case NT_CLEAR_TASK_SET:
5963                 case NT_LUN_RESET:
5964                 case NT_TARGET_RESET:
5965                         /*
5966                          * These are task management functions.
5967                          */
5968                         isp_handle_platform_target_tmf(isp, notify);
5969                         break;
5970                 case NT_BUS_RESET:
5971                 case NT_LIP_RESET:
5972                 case NT_LINK_UP:
5973                 case NT_LINK_DOWN:
5974                         /*
5975                          * No action need be taken here.
5976                          */
5977                         break;
5978                 case NT_HBA_RESET:
5979                         isp_del_all_wwn_entries(isp, ISP_NOCHAN);
5980                         break;
5981                 case NT_GLOBAL_LOGOUT:
5982                 case NT_LOGOUT:
5983                         /*
5984                          * This is device arrival/departure notification
5985                          */
5986                         isp_handle_platform_target_notify_ack(isp, notify);
5987                         break;
5988                 case NT_ARRIVED:
5989                 {
5990                         struct ac_contract ac;
5991                         struct ac_device_changed *fc;
5992
5993                         ac.contract_number = AC_CONTRACT_DEV_CHG;
5994                         fc = (struct ac_device_changed *) ac.contract_data;
5995                         fc->wwpn = notify->nt_wwn;
5996                         fc->port = notify->nt_sid;
5997                         fc->target = notify->nt_nphdl;
5998                         fc->arrived = 1;
5999                         xpt_async(AC_CONTRACT, ISP_FC_PC(isp, notify->nt_channel)->path, &ac);
6000                         break;
6001                 }
6002                 case NT_DEPARTED:
6003                 {
6004                         struct ac_contract ac;
6005                         struct ac_device_changed *fc;
6006
6007                         ac.contract_number = AC_CONTRACT_DEV_CHG;
6008                         fc = (struct ac_device_changed *) ac.contract_data;
6009                         fc->wwpn = notify->nt_wwn;
6010                         fc->port = notify->nt_sid;
6011                         fc->target = notify->nt_nphdl;
6012                         fc->arrived = 0;
6013                         xpt_async(AC_CONTRACT, ISP_FC_PC(isp, notify->nt_channel)->path, &ac);
6014                         break;
6015                 }
6016                 default:
6017                         isp_prt(isp, ISP_LOGALL, "target notify code 0x%x", notify->nt_ncode);
6018                         isp_handle_platform_target_notify_ack(isp, notify);
6019                         break;
6020                 }
6021                 break;
6022         }
6023         case ISPASYNC_TARGET_NOTIFY_ACK:
6024         {
6025                 void *inot;
6026                 va_start(ap, cmd);
6027                 inot = va_arg(ap, void *);
6028                 va_end(ap);
6029                 if (isp_notify_ack(isp, inot)) {
6030                         isp_tna_t *tp = malloc(sizeof (*tp), M_DEVBUF, M_NOWAIT);
6031                         if (tp) {
6032                                 tp->isp = isp;
6033                                 if (inot) {
6034                                         memcpy(tp->data, inot, sizeof (tp->data));
6035                                         tp->not = tp->data;
6036                                 } else {
6037                                         tp->not = NULL;
6038                                 }
6039                                 callout_init_mtx(&tp->timer, &isp->isp_lock, 0);
6040                                 callout_reset(&tp->timer, 5,
6041                                     isp_refire_notify_ack, tp);
6042                         } else {
6043                                 isp_prt(isp, ISP_LOGERR, "you lose- cannot allocate a notify refire");
6044                         }
6045                 }
6046                 break;
6047         }
6048         case ISPASYNC_TARGET_ACTION:
6049         {
6050                 isphdr_t *hp;
6051
6052                 va_start(ap, cmd);
6053                 hp = va_arg(ap, isphdr_t *);
6054                 va_end(ap);
6055                 switch (hp->rqs_entry_type) {
6056                 default:
6057                         isp_prt(isp, ISP_LOGWARN, "%s: unhandled target action 0x%x", __func__, hp->rqs_entry_type);
6058                         break;
6059                 case RQSTYPE_NOTIFY:
6060                         if (IS_SCSI(isp)) {
6061                                 isp_handle_platform_notify_scsi(isp, (in_entry_t *) hp);
6062                         } else if (IS_24XX(isp)) {
6063                                 isp_handle_platform_notify_24xx(isp, (in_fcentry_24xx_t *) hp);
6064                         } else {
6065                                 isp_handle_platform_notify_fc(isp, (in_fcentry_t *) hp);
6066                         }
6067                         break;
6068                 case RQSTYPE_ATIO:
6069                         if (IS_24XX(isp)) {
6070                                 isp_handle_platform_atio7(isp, (at7_entry_t *) hp);
6071                         } else {
6072                                 isp_handle_platform_atio(isp, (at_entry_t *) hp);
6073                         }
6074                         break;
6075                 case RQSTYPE_ATIO2:
6076                         isp_handle_platform_atio2(isp, (at2_entry_t *) hp);
6077                         break;
6078                 case RQSTYPE_CTIO7:
6079                 case RQSTYPE_CTIO3:
6080                 case RQSTYPE_CTIO2:
6081                 case RQSTYPE_CTIO:
6082                         isp_handle_platform_ctio(isp, hp);
6083                         break;
6084                 case RQSTYPE_ABTS_RCVD:
6085                 {
6086                         abts_t *abts = (abts_t *)hp;
6087                         isp_notify_t notify, *nt = &notify;
6088                         tstate_t *tptr;
6089                         fcportdb_t *lp;
6090                         uint16_t chan;
6091                         uint32_t sid, did;
6092
6093                         did = (abts->abts_did_hi << 16) | abts->abts_did_lo;
6094                         sid = (abts->abts_sid_hi << 16) | abts->abts_sid_lo;
6095                         ISP_MEMZERO(nt, sizeof (isp_notify_t));
6096
6097                         nt->nt_hba = isp;
6098                         nt->nt_did = did;
6099                         nt->nt_nphdl = abts->abts_nphdl;
6100                         nt->nt_sid = sid;
6101                         isp_find_chan_by_did(isp, did, &chan);
6102                         if (chan == ISP_NOCHAN) {
6103                                 nt->nt_tgt = TGT_ANY;
6104                         } else {
6105                                 nt->nt_tgt = FCPARAM(isp, chan)->isp_wwpn;
6106                                 if (isp_find_pdb_by_loopid(isp, chan, abts->abts_nphdl, &lp)) {
6107                                         nt->nt_wwn = lp->port_wwn;
6108                                 } else {
6109                                         nt->nt_wwn = INI_ANY;
6110                                 }
6111                         }
6112                         /*
6113                          * Try hard to find the lun for this command.
6114                          */
6115                         tptr = get_lun_statep_from_tag(isp, chan, abts->abts_rxid_task);
6116                         if (tptr) {
6117                                 nt->nt_lun = tptr->ts_lun;
6118                                 rls_lun_statep(isp, tptr);
6119                         } else {
6120                                 nt->nt_lun = LUN_ANY;
6121                         }
6122                         nt->nt_need_ack = 1;
6123                         nt->nt_tagval = abts->abts_rxid_task;
6124                         nt->nt_tagval |= (((uint64_t) abts->abts_rxid_abts) << 32);
6125                         if (abts->abts_rxid_task == ISP24XX_NO_TASK) {
6126                                 isp_prt(isp, ISP_LOGTINFO, "[0x%x] ABTS from N-Port handle 0x%x Port 0x%06x has no task id (rx_id 0x%04x ox_id 0x%04x)",
6127                                     abts->abts_rxid_abts, abts->abts_nphdl, sid, abts->abts_rx_id, abts->abts_ox_id);
6128                         } else {
6129                                 isp_prt(isp, ISP_LOGTINFO, "[0x%x] ABTS from N-Port handle 0x%x Port 0x%06x for task 0x%x (rx_id 0x%04x ox_id 0x%04x)",
6130                                     abts->abts_rxid_abts, abts->abts_nphdl, sid, abts->abts_rxid_task, abts->abts_rx_id, abts->abts_ox_id);
6131                         }
6132                         nt->nt_channel = chan;
6133                         nt->nt_ncode = NT_ABORT_TASK;
6134                         nt->nt_lreserved = hp;
6135                         isp_handle_platform_target_tmf(isp, nt);
6136                         break;
6137                 }
6138                 case RQSTYPE_ENABLE_LUN:
6139                 case RQSTYPE_MODIFY_LUN:
6140                         isp_ledone(isp, (lun_entry_t *) hp);
6141                         break;
6142                 }
6143                 break;
6144         }
6145 #endif
6146         case ISPASYNC_FW_CRASH:
6147         {
6148                 uint16_t mbox1, mbox6;
6149                 mbox1 = ISP_READ(isp, OUTMAILBOX1);
6150                 if (IS_DUALBUS(isp)) {
6151                         mbox6 = ISP_READ(isp, OUTMAILBOX6);
6152                 } else {
6153                         mbox6 = 0;
6154                 }
6155                 isp_prt(isp, ISP_LOGERR, "Internal Firmware Error on bus %d @ RISC Address 0x%x", mbox6, mbox1);
6156                 mbox1 = isp->isp_osinfo.mbox_sleep_ok;
6157                 isp->isp_osinfo.mbox_sleep_ok = 0;
6158                 isp_reinit(isp, 1);
6159                 isp->isp_osinfo.mbox_sleep_ok = mbox1;
6160                 isp_async(isp, ISPASYNC_FW_RESTARTED, NULL);
6161                 break;
6162         }
6163         default:
6164                 isp_prt(isp, ISP_LOGERR, "unknown isp_async event %d", cmd);
6165                 break;
6166         }
6167 }
6168
6169
6170 /*
6171  * Locks are held before coming here.
6172  */
6173 void
6174 isp_uninit(ispsoftc_t *isp)
6175 {
6176         if (IS_24XX(isp)) {
6177                 ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_RESET);
6178         } else {
6179                 ISP_WRITE(isp, HCCR, HCCR_CMD_RESET);
6180         }
6181         ISP_DISABLE_INTS(isp);
6182 }
6183
6184 /*
6185  * When we want to get the 'default' WWNs (when lacking NVRAM), we pick them
6186  * up from our platform default (defww{p|n}n) and morph them based upon
6187  * channel.
6188  *
6189  * When we want to get the 'active' WWNs, we get NVRAM WWNs and then morph them
6190  * based upon channel.
6191  */
6192
6193 uint64_t
6194 isp_default_wwn(ispsoftc_t * isp, int chan, int isactive, int iswwnn)
6195 {
6196         uint64_t seed;
6197         struct isp_fc *fc = ISP_FC_PC(isp, chan);
6198
6199         /*
6200          * If we're asking for a active WWN, the default overrides get
6201          * returned, otherwise the NVRAM value is picked.
6202          *
6203          * If we're asking for a default WWN, we just pick the default override.
6204          */
6205         if (isactive) {
6206                 seed = iswwnn ? fc->def_wwnn : fc->def_wwpn;
6207                 if (seed) {
6208                         return (seed);
6209                 }
6210                 seed = iswwnn ? FCPARAM(isp, chan)->isp_wwnn_nvram : FCPARAM(isp, chan)->isp_wwpn_nvram;
6211                 if (seed) {
6212                         return (seed);
6213                 }
6214                 return (0x400000007F000009ull);
6215         }
6216
6217         seed = iswwnn ? fc->def_wwnn : fc->def_wwpn;
6218
6219         /*
6220          * For channel zero just return what we have. For either ACTIVE or
6221          * DEFAULT cases, we depend on default override of NVRAM values for
6222          * channel zero.
6223          */
6224         if (chan == 0) {
6225                 return (seed);
6226         }
6227
6228         /*
6229          * For other channels, we are doing one of three things:
6230          *
6231          * 1. If what we have now is non-zero, return it. Otherwise we morph
6232          * values from channel 0. 2. If we're here for a WWPN we synthesize
6233          * it if Channel 0's wwpn has a type 2 NAA. 3. If we're here for a
6234          * WWNN we synthesize it if Channel 0's wwnn has a type 2 NAA.
6235          */
6236
6237         if (seed) {
6238                 return (seed);
6239         }
6240         seed = iswwnn ? ISP_FC_PC(isp, 0)->def_wwnn : ISP_FC_PC(isp, 0)->def_wwpn;
6241         if (seed == 0)
6242                 seed = iswwnn ? FCPARAM(isp, 0)->isp_wwnn_nvram : FCPARAM(isp, 0)->isp_wwpn_nvram;
6243
6244         if (((seed >> 60) & 0xf) == 2) {
6245                 /*
6246                  * The type 2 NAA fields for QLogic cards appear be laid out
6247                  * thusly:
6248                  *
6249                  * bits 63..60 NAA == 2 bits 59..57 unused/zero bit 56
6250                  * port (1) or node (0) WWN distinguishor bit 48
6251                  * physical port on dual-port chips (23XX/24XX)
6252                  *
6253                  * This is somewhat nutty, particularly since bit 48 is
6254                  * irrelevant as they assign separate serial numbers to
6255                  * different physical ports anyway.
6256                  *
6257                  * We'll stick our channel number plus one first into bits
6258                  * 57..59 and thence into bits 52..55 which allows for 8 bits
6259                  * of channel which is comfortably more than our maximum
6260                  * (126) now.
6261                  */
6262                 seed &= ~0x0FF0000000000000ULL;
6263                 if (iswwnn == 0) {
6264                         seed |= ((uint64_t) (chan + 1) & 0xf) << 56;
6265                         seed |= ((uint64_t) ((chan + 1) >> 4) & 0xf) << 52;
6266                 }
6267         } else {
6268                 seed = 0;
6269         }
6270         return (seed);
6271 }
6272
6273 void
6274 isp_prt(ispsoftc_t *isp, int level, const char *fmt, ...)
6275 {
6276         int loc;
6277         char lbuf[200];
6278         va_list ap;
6279
6280         if (level != ISP_LOGALL && (level & isp->isp_dblev) == 0) {
6281                 return;
6282         }
6283         snprintf(lbuf, sizeof (lbuf), "%s: ", device_get_nameunit(isp->isp_dev));
6284         loc = strlen(lbuf);
6285         va_start(ap, fmt);
6286         vsnprintf(&lbuf[loc], sizeof (lbuf) - loc - 1, fmt, ap);
6287         va_end(ap);
6288         printf("%s\n", lbuf);
6289 }
6290
6291 void
6292 isp_xs_prt(ispsoftc_t *isp, XS_T *xs, int level, const char *fmt, ...)
6293 {
6294         va_list ap;
6295         if (level != ISP_LOGALL && (level & isp->isp_dblev) == 0) {
6296                 return;
6297         }
6298         xpt_print_path(xs->ccb_h.path);
6299         va_start(ap, fmt);
6300         vprintf(fmt, ap);
6301         va_end(ap);
6302         printf("\n");
6303 }
6304
6305 uint64_t
6306 isp_nanotime_sub(struct timespec *b, struct timespec *a)
6307 {
6308         uint64_t elapsed;
6309         struct timespec x = *b;
6310         timespecsub(&x, a);
6311         elapsed = GET_NANOSEC(&x);
6312         if (elapsed == 0)
6313                 elapsed++;
6314         return (elapsed);
6315 }
6316
6317 int
6318 isp_mbox_acquire(ispsoftc_t *isp)
6319 {
6320         if (isp->isp_osinfo.mboxbsy) {
6321                 return (1);
6322         } else {
6323                 isp->isp_osinfo.mboxcmd_done = 0;
6324                 isp->isp_osinfo.mboxbsy = 1;
6325                 return (0);
6326         }
6327 }
6328
6329 void
6330 isp_mbox_wait_complete(ispsoftc_t *isp, mbreg_t *mbp)
6331 {
6332         unsigned int usecs = mbp->timeout;
6333         unsigned int max, olim, ilim;
6334
6335         if (usecs == 0) {
6336                 usecs = MBCMD_DEFAULT_TIMEOUT;
6337         }
6338         max = isp->isp_mbxwrk0 + 1;
6339
6340         if (isp->isp_osinfo.mbox_sleep_ok) {
6341                 unsigned int ms = (usecs + 999) / 1000;
6342
6343                 isp->isp_osinfo.mbox_sleep_ok = 0;
6344                 isp->isp_osinfo.mbox_sleeping = 1;
6345                 for (olim = 0; olim < max; olim++) {
6346                         msleep(&isp->isp_mbxworkp, &isp->isp_osinfo.lock, PRIBIO, "ispmbx_sleep", isp_mstohz(ms));
6347                         if (isp->isp_osinfo.mboxcmd_done) {
6348                                 break;
6349                         }
6350                 }
6351                 isp->isp_osinfo.mbox_sleep_ok = 1;
6352                 isp->isp_osinfo.mbox_sleeping = 0;
6353         } else {
6354                 for (olim = 0; olim < max; olim++) {
6355                         for (ilim = 0; ilim < usecs; ilim += 100) {
6356                                 uint32_t isr;
6357                                 uint16_t sema, mbox;
6358                                 if (isp->isp_osinfo.mboxcmd_done) {
6359                                         break;
6360                                 }
6361                                 if (ISP_READ_ISR(isp, &isr, &sema, &mbox)) {
6362                                         isp_intr(isp, isr, sema, mbox);
6363                                         if (isp->isp_osinfo.mboxcmd_done) {
6364                                                 break;
6365                                         }
6366                                 }
6367                                 ISP_DELAY(100);
6368                         }
6369                         if (isp->isp_osinfo.mboxcmd_done) {
6370                                 break;
6371                         }
6372                 }
6373         }
6374         if (isp->isp_osinfo.mboxcmd_done == 0) {
6375                 isp_prt(isp, ISP_LOGWARN, "%s Mailbox Command (0x%x) Timeout (%uus) (started @ %s:%d)",
6376                     isp->isp_osinfo.mbox_sleep_ok? "Interrupting" : "Polled", isp->isp_lastmbxcmd, usecs, mbp->func, mbp->lineno);
6377                 mbp->param[0] = MBOX_TIMEOUT;
6378                 isp->isp_osinfo.mboxcmd_done = 1;
6379         }
6380 }
6381
6382 void
6383 isp_mbox_notify_done(ispsoftc_t *isp)
6384 {
6385         if (isp->isp_osinfo.mbox_sleeping) {
6386                 wakeup(&isp->isp_mbxworkp);
6387         }
6388         isp->isp_osinfo.mboxcmd_done = 1;
6389 }
6390
6391 void
6392 isp_mbox_release(ispsoftc_t *isp)
6393 {
6394         isp->isp_osinfo.mboxbsy = 0;
6395 }
6396
6397 int
6398 isp_fc_scratch_acquire(ispsoftc_t *isp, int chan)
6399 {
6400         int ret = 0;
6401         if (isp->isp_osinfo.pc.fc[chan].fcbsy) {
6402                 ret = -1;
6403         } else {
6404                 isp->isp_osinfo.pc.fc[chan].fcbsy = 1;
6405         }
6406         return (ret);
6407 }
6408
6409 int
6410 isp_mstohz(int ms)
6411 {
6412         int hz;
6413         struct timeval t;
6414         t.tv_sec = ms / 1000;
6415         t.tv_usec = (ms % 1000) * 1000;
6416         hz = tvtohz(&t);
6417         if (hz < 0) {
6418                 hz = 0x7fffffff;
6419         }
6420         if (hz == 0) {
6421                 hz = 1;
6422         }
6423         return (hz);
6424 }
6425
6426 void
6427 isp_platform_intr(void *arg)
6428 {
6429         ispsoftc_t *isp = arg;
6430         uint32_t isr;
6431         uint16_t sema, mbox;
6432
6433         ISP_LOCK(isp);
6434         isp->isp_intcnt++;
6435         if (ISP_READ_ISR(isp, &isr, &sema, &mbox) == 0) {
6436                 isp->isp_intbogus++;
6437         } else {
6438                 isp_intr(isp, isr, sema, mbox);
6439         }
6440         ISP_UNLOCK(isp);
6441 }
6442
6443 void
6444 isp_common_dmateardown(ispsoftc_t *isp, struct ccb_scsiio *csio, uint32_t hdl)
6445 {
6446         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
6447                 bus_dmamap_sync(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap, BUS_DMASYNC_POSTREAD);
6448         } else {
6449                 bus_dmamap_sync(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap, BUS_DMASYNC_POSTWRITE);
6450         }
6451         bus_dmamap_unload(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap);
6452 }
6453
6454 /*
6455  * Reset the command reference number for all LUNs on a specific target
6456  * (needed when a target arrives again) or for all targets on a port
6457  * (needed for events like a LIP).
6458  */
6459 void
6460 isp_fcp_reset_crn(struct isp_fc *fc, uint32_t tgt, int tgt_set)
6461 {
6462         int i;
6463         struct isp_nexus *nxp;
6464
6465         if (tgt_set == 0)
6466                 isp_prt(fc->isp, ISP_LOG_SANCFG, "resetting CRN on all targets");
6467         else
6468                 isp_prt(fc->isp, ISP_LOG_SANCFG, "resetting CRN target %u", tgt);
6469
6470         for (i = 0; i < NEXUS_HASH_WIDTH; i++) {
6471                 nxp = fc->nexus_hash[i];
6472                 while (nxp) {
6473                         if ((tgt_set != 0) && (tgt == nxp->tgt))
6474                                 nxp->crnseed = 0;
6475
6476                         nxp = nxp->next;
6477                 }
6478         }
6479 }
6480
6481 int
6482 isp_fcp_next_crn(ispsoftc_t *isp, uint8_t *crnp, XS_T *cmd)
6483 {
6484         uint32_t chan, tgt, lun;
6485         struct isp_fc *fc;
6486         struct isp_nexus *nxp;
6487         int idx;
6488
6489         if (isp->isp_type < ISP_HA_FC_2300)
6490                 return (0);
6491
6492         chan = XS_CHANNEL(cmd);
6493         tgt = XS_TGT(cmd);
6494         lun = XS_LUN(cmd);
6495         fc = &isp->isp_osinfo.pc.fc[chan];
6496         idx = NEXUS_HASH(tgt, lun);
6497         nxp = fc->nexus_hash[idx];
6498
6499         while (nxp) {
6500                 if (nxp->tgt == tgt && nxp->lun == lun)
6501                         break;
6502                 nxp = nxp->next;
6503         }
6504         if (nxp == NULL) {
6505                 nxp = fc->nexus_free_list;
6506                 if (nxp == NULL) {
6507                         nxp = malloc(sizeof (struct isp_nexus), M_DEVBUF, M_ZERO|M_NOWAIT);
6508                         if (nxp == NULL) {
6509                                 return (-1);
6510                         }
6511                 } else {
6512                         fc->nexus_free_list = nxp->next;
6513                 }
6514                 nxp->tgt = tgt;
6515                 nxp->lun = lun;
6516                 nxp->next = fc->nexus_hash[idx];
6517                 fc->nexus_hash[idx] = nxp;
6518         }
6519         if (nxp) {
6520                 if (nxp->crnseed == 0)
6521                         nxp->crnseed = 1;
6522                 if (cmd)
6523                         PISP_PCMD(cmd)->crn = nxp->crnseed;
6524                 *crnp = nxp->crnseed++;
6525                 return (0);
6526         }
6527         return (-1);
6528 }
6529
6530 /*
6531  * We enter with the lock held
6532  */
6533 void
6534 isp_timer(void *arg)
6535 {
6536         ispsoftc_t *isp = arg;
6537 #ifdef  ISP_TARGET_MODE
6538         isp_tmcmd_restart(isp);
6539 #endif
6540         callout_reset(&isp->isp_osinfo.tmo, isp_timer_count, isp_timer, isp);
6541 }
6542
6543 isp_ecmd_t *
6544 isp_get_ecmd(ispsoftc_t *isp)
6545 {
6546         isp_ecmd_t *ecmd = isp->isp_osinfo.ecmd_free;
6547         if (ecmd) {
6548                 isp->isp_osinfo.ecmd_free = ecmd->next;
6549         }
6550         return (ecmd);
6551 }
6552
6553 void
6554 isp_put_ecmd(ispsoftc_t *isp, isp_ecmd_t *ecmd)
6555 {
6556         ecmd->next = isp->isp_osinfo.ecmd_free;
6557         isp->isp_osinfo.ecmd_free = ecmd;
6558 }