sys/dev/isp/isp_freebsd.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009-2020 Alexander Motin <mav@FreeBSD.org>
   5  * Copyright (c) 1997-2009 by Matthew Jacob
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice immediately at the beginning of the file, without modification,
  13  *    this list of conditions, and the following disclaimer.
  14  * 2. The name of the author may not be used to endorse or promote products
  15  *    derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  21  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 /*
  31  * Platform (FreeBSD) dependent common attachment code for Qlogic adapters.
  32  */
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <dev/isp/isp_freebsd.h>
  37 #include <sys/unistd.h>
  38 #include <sys/kthread.h>
  39 #include <sys/conf.h>
  40 #include <sys/module.h>
  41 #include <sys/ioccom.h>
  42 #include <dev/isp/isp_ioctl.h>
  43 #include <sys/devicestat.h>
  44 #include <cam/cam_periph.h>
  45 #include <cam/cam_xpt_periph.h>
  46
  47 MODULE_VERSION(isp, 1);
  48 MODULE_DEPEND(isp, cam, 1, 1, 1);
  49 int isp_announced = 0;
  50 int isp_loop_down_limit = 60;   /* default loop down limit */
  51 int isp_quickboot_time = 7;     /* don't wait more than N secs for loop up */
  52 int isp_gone_device_time = 30;  /* grace time before reporting device lost */
  53 static const char prom3[] = "Chan %d [%u] PortID 0x%06x Departed because of %s";
  54
  55 static void isp_freeze_loopdown(ispsoftc_t *, int);
  56 static void isp_loop_changed(ispsoftc_t *isp, int chan);
  57 static void isp_rq_check_above(ispsoftc_t *);
  58 static void isp_rq_check_below(ispsoftc_t *);
  59 static d_ioctl_t ispioctl;
  60 static void isp_poll(struct cam_sim *);
  61 static callout_func_t isp_watchdog;
  62 static callout_func_t isp_gdt;
  63 static task_fn_t isp_gdt_task;
  64 static void isp_kthread(void *);
  65 static void isp_action(struct cam_sim *, union ccb *);
  66 static int isp_timer_count;
  67 static void isp_timer(void *);
  68
  69 static struct cdevsw isp_cdevsw = {
  70         .d_version =    D_VERSION,
  71         .d_ioctl =      ispioctl,
  72         .d_name =       "isp",
  73 };
  74
  75 static int
  76 isp_role_sysctl(SYSCTL_HANDLER_ARGS)
  77 {
  78         ispsoftc_t *isp = (ispsoftc_t *)arg1;
  79         int chan = arg2;
  80         int error, old, value;
  81
  82         value = FCPARAM(isp, chan)->role;
  83
  84         error = sysctl_handle_int(oidp, &value, 0, req);
  85         if ((error != 0) || (req->newptr == NULL))
  86                 return (error);
  87
  88         if (value < ISP_ROLE_NONE || value > ISP_ROLE_BOTH)
  89                 return (EINVAL);
  90
  91         ISP_LOCK(isp);
  92         old = FCPARAM(isp, chan)->role;
  93
  94         /* We don't allow target mode switch from here. */
  95         value = (old & ISP_ROLE_TARGET) | (value & ISP_ROLE_INITIATOR);
  96
  97         /* If nothing has changed -- we are done. */
  98         if (value == old) {
  99                 ISP_UNLOCK(isp);
 100                 return (0);
 101         }
 102
 103         /* Actually change the role. */
 104         error = isp_control(isp, ISPCTL_CHANGE_ROLE, chan, value);
 105         ISP_UNLOCK(isp);
 106         return (error);
 107 }
 108
 109 static int
 110 isp_attach_chan(ispsoftc_t *isp, struct cam_devq *devq, int chan)
 111 {
 112         struct cam_sim *sim;
 113         struct cam_path *path;
 114 #ifdef  ISP_TARGET_MODE
 115         int i;
 116 #endif
 117
 118         sim = cam_sim_alloc(isp_action, isp_poll, "isp", isp,
 119             device_get_unit(isp->isp_dev), &isp->isp_lock,
 120             isp->isp_maxcmds, isp->isp_maxcmds, devq);
 121         if (sim == NULL)
 122                 return (ENOMEM);
 123
 124         ISP_LOCK(isp);
 125         if (xpt_bus_register(sim, isp->isp_dev, chan) != CAM_SUCCESS) {
 126                 ISP_UNLOCK(isp);
 127                 cam_sim_free(sim, FALSE);
 128                 return (EIO);
 129         }
 130         ISP_UNLOCK(isp);
 131         if (xpt_create_path(&path, NULL, cam_sim_path(sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 132                 ISP_LOCK(isp);
 133                 xpt_bus_deregister(cam_sim_path(sim));
 134                 ISP_UNLOCK(isp);
 135                 cam_sim_free(sim, FALSE);
 136                 return (ENXIO);
 137         }
 138
 139         fcparam *fcp = FCPARAM(isp, chan);
 140         struct isp_fc *fc = ISP_FC_PC(isp, chan);
 141         struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(isp->isp_osinfo.dev);
 142         struct sysctl_oid *tree = device_get_sysctl_tree(isp->isp_osinfo.dev);
 143         char name[16];
 144
 145         ISP_LOCK(isp);
 146         fc->sim = sim;
 147         fc->path = path;
 148         fc->isp = isp;
 149         fc->ready = 1;
 150         fcp->isp_use_gft_id = 1;
 151         fcp->isp_use_gff_id = 1;
 152
 153         callout_init_mtx(&fc->gdt, &isp->isp_lock, 0);
 154         TASK_INIT(&fc->gtask, 1, isp_gdt_task, fc);
 155 #ifdef  ISP_TARGET_MODE
 156         TAILQ_INIT(&fc->waitq);
 157         STAILQ_INIT(&fc->ntfree);
 158         for (i = 0; i < ATPDPSIZE; i++)
 159                 STAILQ_INSERT_TAIL(&fc->ntfree, &fc->ntpool[i], next);
 160         LIST_INIT(&fc->atfree);
 161         for (i = ATPDPSIZE-1; i >= 0; i--)
 162                 LIST_INSERT_HEAD(&fc->atfree, &fc->atpool[i], next);
 163         for (i = 0; i < ATPDPHASHSIZE; i++)
 164                 LIST_INIT(&fc->atused[i]);
 165 #endif
 166         isp_loop_changed(isp, chan);
 167         ISP_UNLOCK(isp);
 168         if (kproc_create(isp_kthread, fc, &fc->kproc, 0, 0,
 169             "%s_%d", device_get_nameunit(isp->isp_osinfo.dev), chan)) {
 170                 xpt_free_path(fc->path);
 171                 ISP_LOCK(isp);
 172                 xpt_bus_deregister(cam_sim_path(fc->sim));
 173                 ISP_UNLOCK(isp);
 174                 cam_sim_free(fc->sim, FALSE);
 175                 return (ENOMEM);
 176         }
 177         fc->num_threads += 1;
 178         if (chan > 0) {
 179                 snprintf(name, sizeof(name), "chan%d", chan);
 180                 tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(tree),
 181                     OID_AUTO, name, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 182                     "Virtual channel");
 183         }
 184         SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 185             "wwnn", CTLFLAG_RD, &fcp->isp_wwnn,
 186             "World Wide Node Name");
 187         SYSCTL_ADD_QUAD(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 188             "wwpn", CTLFLAG_RD, &fcp->isp_wwpn,
 189             "World Wide Port Name");
 190         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 191             "loop_down_limit", CTLFLAG_RW, &fc->loop_down_limit, 0,
 192             "Loop Down Limit");
 193         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 194             "gone_device_time", CTLFLAG_RW, &fc->gone_device_time, 0,
 195             "Gone Device Time");
 196 #if defined(ISP_TARGET_MODE) && defined(DEBUG)
 197         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 198             "inject_lost_data_frame", CTLFLAG_RW, &fc->inject_lost_data_frame, 0,
 199             "Cause a Lost Frame on a Read");
 200 #endif
 201         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 202             "role", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 203             isp, chan, isp_role_sysctl, "I", "Current role");
 204         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 205             "speed", CTLFLAG_RD, &fcp->isp_gbspeed, 0,
 206             "Connection speed in gigabits");
 207         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 208             "linkstate", CTLFLAG_RD, &fcp->isp_linkstate, 0,
 209             "Link state");
 210         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 211             "fwstate", CTLFLAG_RD, &fcp->isp_fwstate, 0,
 212             "Firmware state");
 213         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 214             "loopstate", CTLFLAG_RD, &fcp->isp_loopstate, 0,
 215             "Loop state");
 216         SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 217             "topo", CTLFLAG_RD, &fcp->isp_topo, 0,
 218             "Connection topology");
 219         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 220             "use_gft_id", CTLFLAG_RWTUN, &fcp->isp_use_gft_id, 0,
 221             "Use GFT_ID during fabric scan");
 222         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 223             "use_gff_id", CTLFLAG_RWTUN, &fcp->isp_use_gff_id, 0,
 224             "Use GFF_ID during fabric scan");
 225         return (0);
 226 }
 227
 228 static void
 229 isp_detach_chan(ispsoftc_t *isp, int chan)
 230 {
 231         struct cam_sim *sim;
 232         struct cam_path *path;
 233         int *num_threads;
 234
 235         ISP_GET_PC(isp, chan, sim, sim);
 236         ISP_GET_PC(isp, chan, path, path);
 237         ISP_GET_PC_ADDR(isp, chan, num_threads, num_threads);
 238
 239         xpt_free_path(path);
 240         xpt_bus_deregister(cam_sim_path(sim));
 241         cam_sim_free(sim, FALSE);
 242
 243         /* Wait for the channel's spawned threads to exit. */
 244         wakeup(isp->isp_osinfo.pc.ptr);
 245         while (*num_threads != 0)
 246                 mtx_sleep(isp, &isp->isp_lock, PRIBIO, "isp_reap", 100);
 247 }
 248
 249 int
 250 isp_attach(ispsoftc_t *isp)
 251 {
 252         const char *nu = device_get_nameunit(isp->isp_osinfo.dev);
 253         int du = device_get_unit(isp->isp_dev);
 254         int chan;
 255
 256         /*
 257          * Create the device queue for our SIM(s).
 258          */
 259         isp->isp_osinfo.devq = cam_simq_alloc(isp->isp_maxcmds);
 260         if (isp->isp_osinfo.devq == NULL) {
 261                 return (EIO);
 262         }
 263
 264         for (chan = 0; chan < isp->isp_nchan; chan++) {
 265                 if (isp_attach_chan(isp, isp->isp_osinfo.devq, chan)) {
 266                         goto unwind;
 267                 }
 268         }
 269
 270         callout_init_mtx(&isp->isp_osinfo.tmo, &isp->isp_lock, 0);
 271         isp_timer_count = hz >> 2;
 272         callout_reset(&isp->isp_osinfo.tmo, isp_timer_count, isp_timer, isp);
 273
 274         isp->isp_osinfo.cdev = make_dev(&isp_cdevsw, du, UID_ROOT, GID_OPERATOR, 0600, "%s", nu);
 275         if (isp->isp_osinfo.cdev) {
 276                 isp->isp_osinfo.cdev->si_drv1 = isp;
 277         }
 278         return (0);
 279
 280 unwind:
 281         while (--chan >= 0) {
 282                 struct cam_sim *sim;
 283                 struct cam_path *path;
 284
 285                 ISP_GET_PC(isp, chan, sim, sim);
 286                 ISP_GET_PC(isp, chan, path, path);
 287                 xpt_free_path(path);
 288                 ISP_LOCK(isp);
 289                 xpt_bus_deregister(cam_sim_path(sim));
 290                 ISP_UNLOCK(isp);
 291                 cam_sim_free(sim, FALSE);
 292         }
 293         cam_simq_free(isp->isp_osinfo.devq);
 294         isp->isp_osinfo.devq = NULL;
 295         return (-1);
 296 }
 297
 298 int
 299 isp_detach(ispsoftc_t *isp)
 300 {
 301         int chan;
 302
 303         if (isp->isp_osinfo.cdev) {
 304                 destroy_dev(isp->isp_osinfo.cdev);
 305                 isp->isp_osinfo.cdev = NULL;
 306         }
 307         ISP_LOCK(isp);
 308         /* Tell spawned threads that we're exiting. */
 309         isp->isp_osinfo.is_exiting = 1;
 310         for (chan = isp->isp_nchan - 1; chan >= 0; chan -= 1)
 311                 isp_detach_chan(isp, chan);
 312         ISP_UNLOCK(isp);
 313         callout_drain(&isp->isp_osinfo.tmo);
 314         cam_simq_free(isp->isp_osinfo.devq);
 315         return (0);
 316 }
 317
 318 static void
 319 isp_freeze_loopdown(ispsoftc_t *isp, int chan)
 320 {
 321         struct isp_fc *fc = ISP_FC_PC(isp, chan);
 322
 323         if (fc->sim == NULL)
 324                 return;
 325         if (fc->simqfrozen == 0) {
 326                 isp_prt(isp, ISP_LOGDEBUG0,
 327                     "Chan %d Freeze simq (loopdown)", chan);
 328                 fc->simqfrozen = SIMQFRZ_LOOPDOWN;
 329                 xpt_hold_boot();
 330                 xpt_freeze_simq(fc->sim, 1);
 331         } else {
 332                 isp_prt(isp, ISP_LOGDEBUG0,
 333                     "Chan %d Mark simq frozen (loopdown)", chan);
 334                 fc->simqfrozen |= SIMQFRZ_LOOPDOWN;
 335         }
 336 }
 337
 338 static void
 339 isp_unfreeze_loopdown(ispsoftc_t *isp, int chan)
 340 {
 341         struct isp_fc *fc = ISP_FC_PC(isp, chan);
 342
 343         if (fc->sim == NULL)
 344                 return;
 345         int wasfrozen = fc->simqfrozen & SIMQFRZ_LOOPDOWN;
 346         fc->simqfrozen &= ~SIMQFRZ_LOOPDOWN;
 347         if (wasfrozen && fc->simqfrozen == 0) {
 348                 isp_prt(isp, ISP_LOGDEBUG0,
 349                     "Chan %d Release simq", chan);
 350                 xpt_release_simq(fc->sim, 1);
 351                 xpt_release_boot();
 352         }
 353 }
 354
 355 /*
 356  * Functions to protect from request queue overflow by freezing SIM queue.
 357  * XXX: freezing only one arbitrary SIM, since they all share the queue.
 358  */
 359 static void
 360 isp_rq_check_above(ispsoftc_t *isp)
 361 {
 362         struct isp_fc *fc = ISP_FC_PC(isp, 0);
 363
 364         if (isp->isp_rqovf || fc->sim == NULL)
 365                 return;
 366         if (!isp_rqentry_avail(isp, QENTRY_MAX)) {
 367                 xpt_freeze_simq(fc->sim, 1);
 368                 isp->isp_rqovf = 1;
 369         }
 370 }
 371
 372 static void
 373 isp_rq_check_below(ispsoftc_t *isp)
 374 {
 375         struct isp_fc *fc = ISP_FC_PC(isp, 0);
 376
 377         if (!isp->isp_rqovf || fc->sim == NULL)
 378                 return;
 379         if (isp_rqentry_avail(isp, QENTRY_MAX)) {
 380                 xpt_release_simq(fc->sim, 0);
 381                 isp->isp_rqovf = 0;
 382         }
 383 }
 384
 385 static int
 386 ispioctl(struct cdev *dev, u_long c, caddr_t addr, int flags, struct thread *td)
 387 {
 388         ispsoftc_t *isp;
 389         int nr, chan, retval = ENOTTY;
 390
 391         isp = dev->si_drv1;
 392
 393         switch (c) {
 394         case ISP_SDBLEV:
 395         {
 396                 int olddblev = isp->isp_dblev;
 397                 isp->isp_dblev = *(int *)addr;
 398                 *(int *)addr = olddblev;
 399                 retval = 0;
 400                 break;
 401         }
 402         case ISP_GETROLE:
 403                 chan = *(int *)addr;
 404                 if (chan < 0 || chan >= isp->isp_nchan) {
 405                         retval = -ENXIO;
 406                         break;
 407                 }
 408                 *(int *)addr = FCPARAM(isp, chan)->role;
 409                 retval = 0;
 410                 break;
 411         case ISP_SETROLE:
 412                 nr = *(int *)addr;
 413                 chan = nr >> 8;
 414                 if (chan < 0 || chan >= isp->isp_nchan) {
 415                         retval = -ENXIO;
 416                         break;
 417                 }
 418                 nr &= 0xff;
 419                 if (nr & ~(ISP_ROLE_INITIATOR|ISP_ROLE_TARGET)) {
 420                         retval = EINVAL;
 421                         break;
 422                 }
 423                 ISP_LOCK(isp);
 424                 *(int *)addr = FCPARAM(isp, chan)->role;
 425                 retval = isp_control(isp, ISPCTL_CHANGE_ROLE, chan, nr);
 426                 ISP_UNLOCK(isp);
 427                 retval = 0;
 428                 break;
 429
 430         case ISP_RESETHBA:
 431                 ISP_LOCK(isp);
 432                 isp_reinit(isp, 0);
 433                 ISP_UNLOCK(isp);
 434                 retval = 0;
 435                 break;
 436
 437         case ISP_RESCAN:
 438                 chan = *(intptr_t *)addr;
 439                 if (chan < 0 || chan >= isp->isp_nchan) {
 440                         retval = -ENXIO;
 441                         break;
 442                 }
 443                 ISP_LOCK(isp);
 444                 if (isp_fc_runstate(isp, chan, 5 * 1000000) != LOOP_READY) {
 445                         retval = EIO;
 446                 } else {
 447                         retval = 0;
 448                 }
 449                 ISP_UNLOCK(isp);
 450                 break;
 451
 452         case ISP_FC_LIP:
 453                 chan = *(intptr_t *)addr;
 454                 if (chan < 0 || chan >= isp->isp_nchan) {
 455                         retval = -ENXIO;
 456                         break;
 457                 }
 458                 ISP_LOCK(isp);
 459                 if (isp_control(isp, ISPCTL_SEND_LIP, chan)) {
 460                         retval = EIO;
 461                 } else {
 462                         retval = 0;
 463                 }
 464                 ISP_UNLOCK(isp);
 465                 break;
 466         case ISP_FC_GETDINFO:
 467         {
 468                 struct isp_fc_device *ifc = (struct isp_fc_device *) addr;
 469                 fcportdb_t *lp;
 470
 471                 if (ifc->loopid >= MAX_FC_TARG) {
 472                         retval = EINVAL;
 473                         break;
 474                 }
 475                 lp = &FCPARAM(isp, ifc->chan)->portdb[ifc->loopid];
 476                 if (lp->state != FC_PORTDB_STATE_NIL) {
 477                         ifc->role = (lp->prli_word3 & SVC3_ROLE_MASK) >> SVC3_ROLE_SHIFT;
 478                         ifc->loopid = lp->handle;
 479                         ifc->portid = lp->portid;
 480                         ifc->node_wwn = lp->node_wwn;
 481                         ifc->port_wwn = lp->port_wwn;
 482                         retval = 0;
 483                 } else {
 484                         retval = ENODEV;
 485                 }
 486                 break;
 487         }
 488         case ISP_FC_GETHINFO:
 489         {
 490                 struct isp_hba_device *hba = (struct isp_hba_device *) addr;
 491                 int chan = hba->fc_channel;
 492
 493                 if (chan < 0 || chan >= isp->isp_nchan) {
 494                         retval = ENXIO;
 495                         break;
 496                 }
 497                 hba->fc_fw_major = ISP_FW_MAJORX(isp->isp_fwrev);
 498                 hba->fc_fw_minor = ISP_FW_MINORX(isp->isp_fwrev);
 499                 hba->fc_fw_micro = ISP_FW_MICROX(isp->isp_fwrev);
 500                 hba->fc_nchannels = isp->isp_nchan;
 501                 hba->fc_nports = MAX_FC_TARG;
 502                 hba->fc_speed = FCPARAM(isp, hba->fc_channel)->isp_gbspeed;
 503                 hba->fc_topology = FCPARAM(isp, chan)->isp_topo + 1;
 504                 hba->fc_loopid = FCPARAM(isp, chan)->isp_loopid;
 505                 hba->nvram_node_wwn = FCPARAM(isp, chan)->isp_wwnn_nvram;
 506                 hba->nvram_port_wwn = FCPARAM(isp, chan)->isp_wwpn_nvram;
 507                 hba->active_node_wwn = FCPARAM(isp, chan)->isp_wwnn;
 508                 hba->active_port_wwn = FCPARAM(isp, chan)->isp_wwpn;
 509                 retval = 0;
 510                 break;
 511         }
 512         case ISP_TSK_MGMT:
 513         {
 514                 int needmarker;
 515                 struct isp_fc_tsk_mgmt *fct = (struct isp_fc_tsk_mgmt *) addr;
 516                 uint16_t nphdl;
 517                 void *reqp;
 518                 uint8_t resp[QENTRY_LEN];
 519                 isp24xx_tmf_t tmf;
 520                 isp24xx_statusreq_t sp;
 521                 fcparam *fcp;
 522                 fcportdb_t *lp;
 523                 int i;
 524
 525                 chan = fct->chan;
 526                 if (chan < 0 || chan >= isp->isp_nchan) {
 527                         retval = -ENXIO;
 528                         break;
 529                 }
 530
 531                 needmarker = retval = 0;
 532                 nphdl = fct->loopid;
 533                 ISP_LOCK(isp);
 534                 fcp = FCPARAM(isp, chan);
 535
 536                 for (i = 0; i < MAX_FC_TARG; i++) {
 537                         lp = &fcp->portdb[i];
 538                         if (lp->handle == nphdl) {
 539                                 break;
 540                         }
 541                 }
 542                 if (i == MAX_FC_TARG) {
 543                         retval = ENXIO;
 544                         ISP_UNLOCK(isp);
 545                         break;
 546                 }
 547                 ISP_MEMZERO(&tmf, sizeof(tmf));
 548                 tmf.tmf_header.rqs_entry_type = RQSTYPE_TSK_MGMT;
 549                 tmf.tmf_header.rqs_entry_count = 1;
 550                 tmf.tmf_nphdl = lp->handle;
 551                 tmf.tmf_delay = 2;
 552                 tmf.tmf_timeout = 4;
 553                 tmf.tmf_tidlo = lp->portid;
 554                 tmf.tmf_tidhi = lp->portid >> 16;
 555                 tmf.tmf_vpidx = ISP_GET_VPIDX(isp, chan);
 556                 tmf.tmf_lun[1] = fct->lun & 0xff;
 557                 if (fct->lun >= 256) {
 558                         tmf.tmf_lun[0] = 0x40 | (fct->lun >> 8);
 559                 }
 560                 switch (fct->action) {
 561                 case IPT_CLEAR_ACA:
 562                         tmf.tmf_flags = ISP24XX_TMF_CLEAR_ACA;
 563                         break;
 564                 case IPT_TARGET_RESET:
 565                         tmf.tmf_flags = ISP24XX_TMF_TARGET_RESET;
 566                         needmarker = 1;
 567                         break;
 568                 case IPT_LUN_RESET:
 569                         tmf.tmf_flags = ISP24XX_TMF_LUN_RESET;
 570                         needmarker = 1;
 571                         break;
 572                 case IPT_CLEAR_TASK_SET:
 573                         tmf.tmf_flags = ISP24XX_TMF_CLEAR_TASK_SET;
 574                         needmarker = 1;
 575                         break;
 576                 case IPT_ABORT_TASK_SET:
 577                         tmf.tmf_flags = ISP24XX_TMF_ABORT_TASK_SET;
 578                         needmarker = 1;
 579                         break;
 580                 default:
 581                         retval = EINVAL;
 582                         break;
 583                 }
 584                 if (retval) {
 585                         ISP_UNLOCK(isp);
 586                         break;
 587                 }
 588
 589                 /* Prepare space for response in memory */
 590                 memset(resp, 0xff, sizeof(resp));
 591                 tmf.tmf_handle = isp_allocate_handle(isp, resp,
 592                     ISP_HANDLE_CTRL);
 593                 if (tmf.tmf_handle == 0) {
 594                         isp_prt(isp, ISP_LOGERR,
 595                             "%s: TMF of Chan %d out of handles",
 596                             __func__, chan);
 597                         ISP_UNLOCK(isp);
 598                         retval = ENOMEM;
 599                         break;
 600                 }
 601
 602                 /* Send request and wait for response. */
 603                 reqp = isp_getrqentry(isp);
 604                 if (reqp == NULL) {
 605                         isp_prt(isp, ISP_LOGERR,
 606                             "%s: TMF of Chan %d out of rqent",
 607                             __func__, chan);
 608                         isp_destroy_handle(isp, tmf.tmf_handle);
 609                         ISP_UNLOCK(isp);
 610                         retval = EIO;
 611                         break;
 612                 }
 613                 isp_put_24xx_tmf(isp, &tmf, (isp24xx_tmf_t *)reqp);
 614                 if (isp->isp_dblev & ISP_LOGDEBUG1)
 615                         isp_print_bytes(isp, "IOCB TMF", QENTRY_LEN, reqp);
 616                 ISP_SYNC_REQUEST(isp);
 617                 if (msleep(resp, &isp->isp_lock, 0, "TMF", 5*hz) == EWOULDBLOCK) {
 618                         isp_prt(isp, ISP_LOGERR,
 619                             "%s: TMF of Chan %d timed out",
 620                             __func__, chan);
 621                         isp_destroy_handle(isp, tmf.tmf_handle);
 622                         ISP_UNLOCK(isp);
 623                         retval = EIO;
 624                         break;
 625                 }
 626                 if (isp->isp_dblev & ISP_LOGDEBUG1)
 627                 isp_print_bytes(isp, "IOCB TMF response", QENTRY_LEN, resp);
 628                 isp_get_24xx_response(isp, (isp24xx_statusreq_t *)resp, &sp);
 629
 630                 if (sp.req_completion_status != 0)
 631                         retval = EIO;
 632                 else if (needmarker)
 633                         fcp->sendmarker = 1;
 634                 ISP_UNLOCK(isp);
 635                 break;
 636         }
 637         default:
 638                 break;
 639         }
 640         return (retval);
 641 }
 642
 643 /*
 644  * Local Inlines
 645  */
 646
 647 static ISP_INLINE int isp_get_pcmd(ispsoftc_t *, union ccb *);
 648 static ISP_INLINE void isp_free_pcmd(ispsoftc_t *, union ccb *);
 649
 650 static ISP_INLINE int
 651 isp_get_pcmd(ispsoftc_t *isp, union ccb *ccb)
 652 {
 653         ISP_PCMD(ccb) = isp->isp_osinfo.pcmd_free;
 654         if (ISP_PCMD(ccb) == NULL) {
 655                 return (-1);
 656         }
 657         isp->isp_osinfo.pcmd_free = ((struct isp_pcmd *)ISP_PCMD(ccb))->next;
 658         return (0);
 659 }
 660
 661 static ISP_INLINE void
 662 isp_free_pcmd(ispsoftc_t *isp, union ccb *ccb)
 663 {
 664         if (ISP_PCMD(ccb)) {
 665 #ifdef  ISP_TARGET_MODE
 666                 PISP_PCMD(ccb)->datalen = 0;
 667 #endif
 668                 PISP_PCMD(ccb)->next = isp->isp_osinfo.pcmd_free;
 669                 isp->isp_osinfo.pcmd_free = ISP_PCMD(ccb);
 670                 ISP_PCMD(ccb) = NULL;
 671         }
 672 }
 673
 674 /*
 675  * Put the target mode functions here, because some are inlines
 676  */
 677 #ifdef  ISP_TARGET_MODE
 678 static ISP_INLINE tstate_t *get_lun_statep(ispsoftc_t *, int, lun_id_t);
 679 static atio_private_data_t *isp_get_atpd(ispsoftc_t *, int, uint32_t);
 680 static atio_private_data_t *isp_find_atpd(ispsoftc_t *, int, uint32_t);
 681 static void isp_put_atpd(ispsoftc_t *, int, atio_private_data_t *);
 682 static inot_private_data_t *isp_get_ntpd(ispsoftc_t *, int);
 683 static inot_private_data_t *isp_find_ntpd(ispsoftc_t *, int, uint32_t, uint32_t);
 684 static void isp_put_ntpd(ispsoftc_t *, int, inot_private_data_t *);
 685 static cam_status create_lun_state(ispsoftc_t *, int, struct cam_path *, tstate_t **);
 686 static void destroy_lun_state(ispsoftc_t *, int, tstate_t *);
 687 static void isp_enable_lun(ispsoftc_t *, union ccb *);
 688 static void isp_disable_lun(ispsoftc_t *, union ccb *);
 689 static callout_func_t isp_refire_notify_ack;
 690 static void isp_complete_ctio(ispsoftc_t *isp, union ccb *);
 691 enum Start_Ctio_How { FROM_CAM, FROM_TIMER, FROM_SRR, FROM_CTIO_DONE };
 692 static void isp_target_start_ctio(ispsoftc_t *, union ccb *, enum Start_Ctio_How);
 693 static void isp_handle_platform_atio7(ispsoftc_t *, at7_entry_t *);
 694 static void isp_handle_platform_ctio(ispsoftc_t *, ct7_entry_t *);
 695 static int isp_handle_platform_target_notify_ack(ispsoftc_t *, isp_notify_t *, uint32_t rsp);
 696 static void isp_handle_platform_target_tmf(ispsoftc_t *, isp_notify_t *);
 697 static void isp_target_mark_aborted_early(ispsoftc_t *, int chan, tstate_t *, uint32_t);
 698
 699 static ISP_INLINE tstate_t *
 700 get_lun_statep(ispsoftc_t *isp, int bus, lun_id_t lun)
 701 {
 702         tstate_t *tptr = NULL;
 703         struct tslist *lhp;
 704
 705         if (bus < isp->isp_nchan) {
 706                 ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp);
 707                 SLIST_FOREACH(tptr, lhp, next) {
 708                         if (tptr->ts_lun == lun)
 709                                 return (tptr);
 710                 }
 711         }
 712         return (NULL);
 713 }
 714
 715 static int
 716 isp_atio_restart(ispsoftc_t *isp, int bus, tstate_t *tptr)
 717 {
 718         inot_private_data_t *ntp;
 719         struct ntpdlist rq;
 720
 721         if (STAILQ_EMPTY(&tptr->restart_queue))
 722                 return (0);
 723         STAILQ_INIT(&rq);
 724         STAILQ_CONCAT(&rq, &tptr->restart_queue);
 725         while ((ntp = STAILQ_FIRST(&rq)) != NULL) {
 726                 STAILQ_REMOVE_HEAD(&rq, next);
 727                 isp_prt(isp, ISP_LOGTDEBUG0,
 728                     "%s: restarting resrc deprived %x", __func__,
 729                     ((at7_entry_t *)ntp->data)->at_rxid);
 730                 isp_handle_platform_atio7(isp, (at7_entry_t *) ntp->data);
 731                 isp_put_ntpd(isp, bus, ntp);
 732                 if (!STAILQ_EMPTY(&tptr->restart_queue))
 733                         break;
 734         }
 735         if (!STAILQ_EMPTY(&rq)) {
 736                 STAILQ_CONCAT(&rq, &tptr->restart_queue);
 737                 STAILQ_CONCAT(&tptr->restart_queue, &rq);
 738         }
 739         return (!STAILQ_EMPTY(&tptr->restart_queue));
 740 }
 741
 742 static void
 743 isp_tmcmd_restart(ispsoftc_t *isp)
 744 {
 745         tstate_t *tptr;
 746         union ccb *ccb;
 747         struct tslist *lhp;
 748         struct isp_ccbq *waitq;
 749         int bus, i;
 750
 751         for (bus = 0; bus < isp->isp_nchan; bus++) {
 752                 for (i = 0; i < LUN_HASH_SIZE; i++) {
 753                         ISP_GET_PC_ADDR(isp, bus, lun_hash[i], lhp);
 754                         SLIST_FOREACH(tptr, lhp, next)
 755                                 isp_atio_restart(isp, bus, tptr);
 756                 }
 757
 758                 /*
 759                  * We only need to do this once per channel.
 760                  */
 761                 ISP_GET_PC_ADDR(isp, bus, waitq, waitq);
 762                 ccb = (union ccb *)TAILQ_FIRST(waitq);
 763                 if (ccb != NULL) {
 764                         TAILQ_REMOVE(waitq, &ccb->ccb_h, sim_links.tqe);
 765                         isp_target_start_ctio(isp, ccb, FROM_TIMER);
 766                 }
 767         }
 768         isp_rq_check_above(isp);
 769         isp_rq_check_below(isp);
 770 }
 771
 772 static atio_private_data_t *
 773 isp_get_atpd(ispsoftc_t *isp, int chan, uint32_t tag)
 774 {
 775         struct atpdlist *atfree;
 776         struct atpdlist *atused;
 777         atio_private_data_t *atp;
 778
 779         ISP_GET_PC_ADDR(isp, chan, atfree, atfree);
 780         atp = LIST_FIRST(atfree);
 781         if (atp) {
 782                 LIST_REMOVE(atp, next);
 783                 atp->tag = tag;
 784                 ISP_GET_PC(isp, chan, atused, atused);
 785                 LIST_INSERT_HEAD(&atused[ATPDPHASH(tag)], atp, next);
 786         }
 787         return (atp);
 788 }
 789
 790 static atio_private_data_t *
 791 isp_find_atpd(ispsoftc_t *isp, int chan, uint32_t tag)
 792 {
 793         struct atpdlist *atused;
 794         atio_private_data_t *atp;
 795
 796         ISP_GET_PC(isp, chan, atused, atused);
 797         LIST_FOREACH(atp, &atused[ATPDPHASH(tag)], next) {
 798                 if (atp->tag == tag)
 799                         return (atp);
 800         }
 801         return (NULL);
 802 }
 803
 804 static void
 805 isp_put_atpd(ispsoftc_t *isp, int chan, atio_private_data_t *atp)
 806 {
 807         struct atpdlist *atfree;
 808
 809         if (atp->ests) {
 810                 isp_put_ecmd(isp, atp->ests);
 811         }
 812         LIST_REMOVE(atp, next);
 813         memset(atp, 0, sizeof (*atp));
 814         ISP_GET_PC_ADDR(isp, chan, atfree, atfree);
 815         LIST_INSERT_HEAD(atfree, atp, next);
 816 }
 817
 818 static void
 819 isp_dump_atpd(ispsoftc_t *isp, int chan)
 820 {
 821         atio_private_data_t *atp, *atpool;
 822         const char *states[8] = { "Free", "ATIO", "CAM", "CTIO", "LAST_CTIO", "PDON", "?6", "7" };
 823
 824         ISP_GET_PC(isp, chan, atpool, atpool);
 825         for (atp = atpool; atp < &atpool[ATPDPSIZE]; atp++) {
 826                 if (atp->state == ATPD_STATE_FREE)
 827                         continue;
 828                 isp_prt(isp, ISP_LOGALL, "Chan %d ATP [0x%x] origdlen %u bytes_xfrd %u lun %jx nphdl 0x%04x s_id 0x%06x d_id 0x%06x oxid 0x%04x state %s",
 829                     chan, atp->tag, atp->orig_datalen, atp->bytes_xfered, (uintmax_t)atp->lun, atp->nphdl, atp->sid, atp->did, atp->oxid, states[atp->state & 0x7]);
 830         }
 831 }
 832
 833 static inot_private_data_t *
 834 isp_get_ntpd(ispsoftc_t *isp, int chan)
 835 {
 836         struct ntpdlist *ntfree;
 837         inot_private_data_t *ntp;
 838
 839         ISP_GET_PC_ADDR(isp, chan, ntfree, ntfree);
 840         ntp = STAILQ_FIRST(ntfree);
 841         if (ntp)
 842                 STAILQ_REMOVE_HEAD(ntfree, next);
 843         return (ntp);
 844 }
 845
 846 static inot_private_data_t *
 847 isp_find_ntpd(ispsoftc_t *isp, int chan, uint32_t tag_id, uint32_t seq_id)
 848 {
 849         inot_private_data_t *ntp, *ntp2;
 850
 851         ISP_GET_PC(isp, chan, ntpool, ntp);
 852         ISP_GET_PC_ADDR(isp, chan, ntpool[ATPDPSIZE], ntp2);
 853         for (; ntp < ntp2; ntp++) {
 854                 if (ntp->tag_id == tag_id && ntp->seq_id == seq_id)
 855                         return (ntp);
 856         }
 857         return (NULL);
 858 }
 859
 860 static void
 861 isp_put_ntpd(ispsoftc_t *isp, int chan, inot_private_data_t *ntp)
 862 {
 863         struct ntpdlist *ntfree;
 864
 865         ntp->tag_id = ntp->seq_id = 0;
 866         ISP_GET_PC_ADDR(isp, chan, ntfree, ntfree);
 867         STAILQ_INSERT_HEAD(ntfree, ntp, next);
 868 }
 869
 870 static cam_status
 871 create_lun_state(ispsoftc_t *isp, int bus, struct cam_path *path, tstate_t **rslt)
 872 {
 873         lun_id_t lun;
 874         struct tslist *lhp;
 875         tstate_t *tptr;
 876
 877         lun = xpt_path_lun_id(path);
 878         tptr = malloc(sizeof (tstate_t), M_DEVBUF, M_NOWAIT|M_ZERO);
 879         if (tptr == NULL) {
 880                 return (CAM_RESRC_UNAVAIL);
 881         }
 882         tptr->ts_lun = lun;
 883         SLIST_INIT(&tptr->atios);
 884         SLIST_INIT(&tptr->inots);
 885         STAILQ_INIT(&tptr->restart_queue);
 886         ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(lun)], lhp);
 887         SLIST_INSERT_HEAD(lhp, tptr, next);
 888         *rslt = tptr;
 889         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, path, "created tstate\n");
 890         return (CAM_REQ_CMP);
 891 }
 892
 893 static void
 894 destroy_lun_state(ispsoftc_t *isp, int bus, tstate_t *tptr)
 895 {
 896         union ccb *ccb;
 897         struct tslist *lhp;
 898         inot_private_data_t *ntp;
 899
 900         while ((ccb = (union ccb *)SLIST_FIRST(&tptr->atios)) != NULL) {
 901                 SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
 902                 ccb->ccb_h.status = CAM_REQ_ABORTED;
 903                 xpt_done(ccb);
 904         };
 905         while ((ccb = (union ccb *)SLIST_FIRST(&tptr->inots)) != NULL) {
 906                 SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle);
 907                 ccb->ccb_h.status = CAM_REQ_ABORTED;
 908                 xpt_done(ccb);
 909         }
 910         while ((ntp = STAILQ_FIRST(&tptr->restart_queue)) != NULL) {
 911                 isp_endcmd(isp, ntp->data, NIL_HANDLE, bus, SCSI_STATUS_BUSY, 0);
 912                 STAILQ_REMOVE_HEAD(&tptr->restart_queue, next);
 913                 isp_put_ntpd(isp, bus, ntp);
 914         }
 915         ISP_GET_PC_ADDR(isp, bus, lun_hash[LUN_HASH_FUNC(tptr->ts_lun)], lhp);
 916         SLIST_REMOVE(lhp, tptr, tstate, next);
 917         free(tptr, M_DEVBUF);
 918 }
 919
 920 static void
 921 isp_enable_lun(ispsoftc_t *isp, union ccb *ccb)
 922 {
 923         tstate_t *tptr;
 924         int bus;
 925         target_id_t target;
 926         lun_id_t lun;
 927
 928         /*
 929          * We only support either target and lun both wildcard
 930          * or target and lun both non-wildcard.
 931          */
 932         bus = XS_CHANNEL(ccb);
 933         target = ccb->ccb_h.target_id;
 934         lun = ccb->ccb_h.target_lun;
 935         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0|ISP_LOGCONFIG, ccb->ccb_h.path,
 936             "enabling lun %jx\n", (uintmax_t)lun);
 937         if ((target == CAM_TARGET_WILDCARD) != (lun == CAM_LUN_WILDCARD)) {
 938                 ccb->ccb_h.status = CAM_LUN_INVALID;
 939                 xpt_done(ccb);
 940                 return;
 941         }
 942
 943         /* Create the state pointer. It should not already exist. */
 944         tptr = get_lun_statep(isp, bus, lun);
 945         if (tptr) {
 946                 ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
 947                 xpt_done(ccb);
 948                 return;
 949         }
 950         ccb->ccb_h.status = create_lun_state(isp, bus, ccb->ccb_h.path, &tptr);
 951         if (ccb->ccb_h.status != CAM_REQ_CMP) {
 952                 xpt_done(ccb);
 953                 return;
 954         }
 955
 956         ccb->ccb_h.status = CAM_REQ_CMP;
 957         xpt_done(ccb);
 958 }
 959
 960 static void
 961 isp_disable_lun(ispsoftc_t *isp, union ccb *ccb)
 962 {
 963         tstate_t *tptr = NULL;
 964         int bus;
 965         target_id_t target;
 966         lun_id_t lun;
 967
 968         bus = XS_CHANNEL(ccb);
 969         target = ccb->ccb_h.target_id;
 970         lun = ccb->ccb_h.target_lun;
 971         ISP_PATH_PRT(isp, ISP_LOGTDEBUG0|ISP_LOGCONFIG, ccb->ccb_h.path,
 972             "disabling lun %jx\n", (uintmax_t)lun);
 973         if ((target == CAM_TARGET_WILDCARD) != (lun == CAM_LUN_WILDCARD)) {
 974                 ccb->ccb_h.status = CAM_LUN_INVALID;
 975                 xpt_done(ccb);
 976                 return;
 977         }
 978
 979         /* Find the state pointer. */
 980         if ((tptr = get_lun_statep(isp, bus, lun)) == NULL) {
 981                 ccb->ccb_h.status = CAM_PATH_INVALID;
 982                 xpt_done(ccb);
 983                 return;
 984         }
 985
 986         destroy_lun_state(isp, bus, tptr);
 987         ccb->ccb_h.status = CAM_REQ_CMP;
 988         xpt_done(ccb);
 989 }
 990
 991 static void
 992 isp_target_start_ctio(ispsoftc_t *isp, union ccb *ccb, enum Start_Ctio_How how)
 993 {
 994         int fctape, sendstatus, resid;
 995         fcparam *fcp;
 996         atio_private_data_t *atp;
 997         struct ccb_scsiio *cso;
 998         struct isp_ccbq *waitq;
 999         uint32_t dmaresult, handle, xfrlen, sense_length, tmp;
1000         ct7_entry_t local, *cto = &local;
1001
1002         isp_prt(isp, ISP_LOGTDEBUG0, "%s: ENTRY[0x%x] how %u xfrlen %u sendstatus %d sense_len %u", __func__, ccb->csio.tag_id, how, ccb->csio.dxfer_len,
1003             (ccb->ccb_h.flags & CAM_SEND_STATUS) != 0, ((ccb->ccb_h.flags & CAM_SEND_SENSE)? ccb->csio.sense_len : 0));
1004
1005         ISP_GET_PC_ADDR(isp, XS_CHANNEL(ccb), waitq, waitq);
1006         switch (how) {
1007         case FROM_CAM:
1008                 /*
1009                  * Insert at the tail of the list, if any, waiting CTIO CCBs
1010                  */
1011                 TAILQ_INSERT_TAIL(waitq, &ccb->ccb_h, sim_links.tqe);
1012                 break;
1013         case FROM_TIMER:
1014         case FROM_SRR:
1015         case FROM_CTIO_DONE:
1016                 TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe);
1017                 break;
1018         }
1019
1020         while ((ccb = (union ccb *) TAILQ_FIRST(waitq)) != NULL) {
1021                 TAILQ_REMOVE(waitq, &ccb->ccb_h, sim_links.tqe);
1022
1023                 cso = &ccb->csio;
1024                 xfrlen = cso->dxfer_len;
1025                 if (xfrlen == 0) {
1026                         if ((ccb->ccb_h.flags & CAM_SEND_STATUS) == 0) {
1027                                 ISP_PATH_PRT(isp, ISP_LOGERR, ccb->ccb_h.path, "a data transfer length of zero but no status to send is wrong\n");
1028                                 ccb->ccb_h.status = CAM_REQ_INVALID;
1029                                 xpt_done(ccb);
1030                                 continue;
1031                         }
1032                 }
1033
1034                 atp = isp_find_atpd(isp, XS_CHANNEL(ccb), cso->tag_id);
1035                 if (atp == NULL) {
1036                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] cannot find private data adjunct in %s", __func__, cso->tag_id, __func__);
1037                         isp_dump_atpd(isp, XS_CHANNEL(ccb));
1038                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1039                         xpt_done(ccb);
1040                         continue;
1041                 }
1042
1043                 /*
1044                  * Is this command a dead duck?
1045                  */
1046                 if (atp->dead) {
1047                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] not sending a CTIO for a dead command", __func__, cso->tag_id);
1048                         ccb->ccb_h.status = CAM_REQ_ABORTED;
1049                         xpt_done(ccb);
1050                         continue;
1051                 }
1052
1053                 /*
1054                  * Check to make sure we're still in target mode.
1055                  */
1056                 fcp = FCPARAM(isp, XS_CHANNEL(ccb));
1057                 if ((fcp->role & ISP_ROLE_TARGET) == 0) {
1058                         isp_prt(isp, ISP_LOGERR, "%s: [0x%x] stopping sending a CTIO because we're no longer in target mode", __func__, cso->tag_id);
1059                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1060                         xpt_done(ccb);
1061                         continue;
1062                 }
1063
1064                 /*
1065                  * We're only handling ATPD_CCB_OUTSTANDING outstanding CCB at a time (one of which
1066                  * could be split into two CTIOs to split data and status).
1067                  */
1068                 if (atp->ctcnt >= ATPD_CCB_OUTSTANDING) {
1069                         isp_prt(isp, ISP_LOGTINFO, "[0x%x] handling only %d CCBs at a time (flags for this ccb: 0x%x)", cso->tag_id, ATPD_CCB_OUTSTANDING, ccb->ccb_h.flags);
1070                         TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe);
1071                         break;
1072                 }
1073
1074                 /*
1075                  * Does the initiator expect FC-Tape style responses?
1076                  */
1077                 if ((atp->word3 & PRLI_WD3_RETRY) && fcp->fctape_enabled) {
1078                         fctape = 1;
1079                 } else {
1080                         fctape = 0;
1081                 }
1082
1083                 /*
1084                  * If we already did the data xfer portion of a CTIO that sends data
1085                  * and status, don't do it again and do the status portion now.
1086                  */
1087                 if (atp->sendst) {
1088                         isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] now sending synthesized status orig_dl=%u xfered=%u bit=%u",
1089                             cso->tag_id, atp->orig_datalen, atp->bytes_xfered, atp->bytes_in_transit);
1090                         xfrlen = 0;     /* we already did the data transfer */
1091                         atp->sendst = 0;
1092                 }
1093                 if (ccb->ccb_h.flags & CAM_SEND_STATUS) {
1094                         sendstatus = 1;
1095                 } else {
1096                         sendstatus = 0;
1097                 }
1098
1099                 if (ccb->ccb_h.flags & CAM_SEND_SENSE) {
1100                         KASSERT((sendstatus != 0), ("how can you have CAM_SEND_SENSE w/o CAM_SEND_STATUS?"));
1101                         /*
1102                          * Sense length is not the entire sense data structure size. Periph
1103                          * drivers don't seem to be setting sense_len to reflect the actual
1104                          * size. We'll peek inside to get the right amount.
1105                          */
1106                         sense_length = cso->sense_len;
1107
1108                         /*
1109                          * This 'cannot' happen
1110                          */
1111                         if (sense_length > (XCMD_SIZE - MIN_FCP_RESPONSE_SIZE)) {
1112                                 sense_length = XCMD_SIZE - MIN_FCP_RESPONSE_SIZE;
1113                         }
1114                 } else {
1115                         sense_length = 0;
1116                 }
1117
1118                 /*
1119                  * Check for overflow
1120                  */
1121                 tmp = atp->bytes_xfered + atp->bytes_in_transit;
1122                 if (xfrlen > 0 && tmp > atp->orig_datalen) {
1123                         isp_prt(isp, ISP_LOGERR,
1124                             "%s: [0x%x] data overflow by %u bytes", __func__,
1125                             cso->tag_id, tmp + xfrlen - atp->orig_datalen);
1126                         ccb->ccb_h.status = CAM_DATA_RUN_ERR;
1127                         xpt_done(ccb);
1128                         continue;
1129                 }
1130                 if (xfrlen > atp->orig_datalen - tmp) {
1131                         xfrlen = atp->orig_datalen - tmp;
1132                         if (xfrlen == 0 && !sendstatus) {
1133                                 cso->resid = cso->dxfer_len;
1134                                 ccb->ccb_h.status = CAM_REQ_CMP;
1135                                 xpt_done(ccb);
1136                                 continue;
1137                         }
1138                 }
1139
1140                 memset(cto, 0, QENTRY_LEN);
1141                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
1142                 cto->ct_header.rqs_entry_count = 1;
1143                 cto->ct_header.rqs_seqno |= ATPD_SEQ_NOTIFY_CAM;
1144                 ATPD_SET_SEQNO(cto, atp);
1145                 cto->ct_nphdl = atp->nphdl;
1146                 cto->ct_rxid = atp->tag;
1147                 cto->ct_iid_lo = atp->sid;
1148                 cto->ct_iid_hi = atp->sid >> 16;
1149                 cto->ct_oxid = atp->oxid;
1150                 cto->ct_vpidx = ISP_GET_VPIDX(isp, XS_CHANNEL(ccb));
1151                 cto->ct_timeout = XS_TIME(ccb);
1152                 cto->ct_flags = atp->tattr << CT7_TASK_ATTR_SHIFT;
1153
1154                 /*
1155                  * Mode 1, status, no data. Only possible when we are sending status, have
1156                  * no data to transfer, and any sense data can fit into a ct7_entry_t.
1157                  *
1158                  * Mode 2, status, no data. We have to use this in the case that
1159                  * the sense data won't fit into a ct7_entry_t.
1160                  *
1161                  */
1162                 if (sendstatus && xfrlen == 0) {
1163                         cto->ct_flags |= CT7_SENDSTATUS | CT7_NO_DATA;
1164                         resid = atp->orig_datalen - atp->bytes_xfered - atp->bytes_in_transit;
1165                         if (sense_length <= MAXRESPLEN_24XX) {
1166                                 cto->ct_flags |= CT7_FLAG_MODE1;
1167                                 cto->ct_scsi_status = cso->scsi_status;
1168                                 if (resid < 0) {
1169                                         cto->ct_resid = -resid;
1170                                         cto->ct_scsi_status |= (FCP_RESID_OVERFLOW << 8);
1171                                 } else if (resid > 0) {
1172                                         cto->ct_resid = resid;
1173                                         cto->ct_scsi_status |= (FCP_RESID_UNDERFLOW << 8);
1174                                 }
1175                                 if (fctape) {
1176                                         cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF;
1177                                 }
1178                                 if (sense_length) {
1179                                         cto->ct_scsi_status |= (FCP_SNSLEN_VALID << 8);
1180                                         cto->rsp.m1.ct_resplen = cto->ct_senselen = sense_length;
1181                                         memcpy(cto->rsp.m1.ct_resp, &cso->sense_data, sense_length);
1182                                 }
1183                         } else {
1184                                 bus_addr_t addr;
1185                                 fcp_rsp_iu_t rp;
1186
1187                                 if (atp->ests == NULL) {
1188                                         atp->ests = isp_get_ecmd(isp);
1189                                         if (atp->ests == NULL) {
1190                                                 TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe);
1191                                                 break;
1192                                         }
1193                                 }
1194                                 memset(&rp, 0, sizeof(rp));
1195                                 if (fctape) {
1196                                         cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF;
1197                                         rp.fcp_rsp_bits |= FCP_CONF_REQ;
1198                                 }
1199                                 cto->ct_flags |= CT7_FLAG_MODE2;
1200                                 rp.fcp_rsp_scsi_status = cso->scsi_status;
1201                                 if (resid < 0) {
1202                                         rp.fcp_rsp_resid = -resid;
1203                                         rp.fcp_rsp_bits |= FCP_RESID_OVERFLOW;
1204                                 } else if (resid > 0) {
1205                                         rp.fcp_rsp_resid = resid;
1206                                         rp.fcp_rsp_bits |= FCP_RESID_UNDERFLOW;
1207                                 }
1208                                 if (sense_length) {
1209                                         rp.fcp_rsp_snslen = sense_length;
1210                                         cto->ct_senselen = sense_length;
1211                                         rp.fcp_rsp_bits |= FCP_SNSLEN_VALID;
1212                                         isp_put_fcp_rsp_iu(isp, &rp, atp->ests);
1213                                         memcpy(((fcp_rsp_iu_t *)atp->ests)->fcp_rsp_extra, &cso->sense_data, sense_length);
1214                                 } else {
1215                                         isp_put_fcp_rsp_iu(isp, &rp, atp->ests);
1216                                 }
1217                                 if (isp->isp_dblev & ISP_LOGTDEBUG1) {
1218                                         isp_print_bytes(isp, "FCP Response Frame After Swizzling", MIN_FCP_RESPONSE_SIZE + sense_length, atp->ests);
1219                                 }
1220                                 bus_dmamap_sync(isp->isp_osinfo.ecmd_dmat, isp->isp_osinfo.ecmd_map, BUS_DMASYNC_PREWRITE);
1221                                 addr = isp->isp_osinfo.ecmd_dma;
1222                                 addr += ((((isp_ecmd_t *)atp->ests) - isp->isp_osinfo.ecmd_base) * XCMD_SIZE);
1223                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: ests base %p vaddr %p ecmd_dma %jx addr %jx len %u", __func__, isp->isp_osinfo.ecmd_base, atp->ests,
1224                                     (uintmax_t) isp->isp_osinfo.ecmd_dma, (uintmax_t)addr, MIN_FCP_RESPONSE_SIZE + sense_length);
1225                                 cto->rsp.m2.ct_datalen = MIN_FCP_RESPONSE_SIZE + sense_length;
1226                                 cto->rsp.m2.ct_fcp_rsp_iudata.ds_base = DMA_LO32(addr);
1227                                 cto->rsp.m2.ct_fcp_rsp_iudata.ds_basehi = DMA_HI32(addr);
1228                                 cto->rsp.m2.ct_fcp_rsp_iudata.ds_count = MIN_FCP_RESPONSE_SIZE + sense_length;
1229                         }
1230                         if (sense_length) {
1231                                 isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d slen %u sense: %x %x/%x/%x", __func__,
1232                                     cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, cto->ct_resid, sense_length,
1233                                     cso->sense_data.error_code, cso->sense_data.sense_buf[1], cso->sense_data.sense_buf[11], cso->sense_data.sense_buf[12]);
1234                         } else {
1235                                 isp_prt(isp, ISP_LOGDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x resid=%d", __func__,
1236                                     cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, cto->ct_resid);
1237                         }
1238                         atp->state = ATPD_STATE_LAST_CTIO;
1239                 }
1240
1241                 /*
1242                  * Mode 0 data transfers, *possibly* with status.
1243                  */
1244                 if (xfrlen != 0) {
1245                         cto->ct_flags |= CT7_FLAG_MODE0;
1246                         if ((cso->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1247                                 cto->ct_flags |= CT7_DATA_IN;
1248                         } else {
1249                                 cto->ct_flags |= CT7_DATA_OUT;
1250                         }
1251
1252                         cto->rsp.m0.reloff = atp->bytes_xfered + atp->bytes_in_transit;
1253                         cto->rsp.m0.ct_xfrlen = xfrlen;
1254
1255 #ifdef  DEBUG
1256                         if (ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame && xfrlen > ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame) {
1257                                 isp_prt(isp, ISP_LOGWARN, "%s: truncating data frame with xfrlen %d to %d", __func__, xfrlen, xfrlen - (xfrlen >> 2));
1258                                 ISP_FC_PC(isp, XS_CHANNEL(ccb))->inject_lost_data_frame = 0;
1259                                 cto->rsp.m0.ct_xfrlen -= xfrlen >> 2;
1260                         }
1261 #endif
1262                         if (sendstatus) {
1263                                 resid = atp->orig_datalen - atp->bytes_xfered - xfrlen;
1264                                 if (cso->scsi_status == SCSI_STATUS_OK && resid == 0 /* && fctape == 0 */) {
1265                                         cto->ct_flags |= CT7_SENDSTATUS;
1266                                         atp->state = ATPD_STATE_LAST_CTIO;
1267                                         if (fctape) {
1268                                                 cto->ct_flags |= CT7_CONFIRM|CT7_EXPLCT_CONF;
1269                                         }
1270                                 } else {
1271                                         atp->sendst = 1;        /* send status later */
1272                                         cto->ct_header.rqs_seqno &= ~ATPD_SEQ_NOTIFY_CAM;
1273                                         atp->state = ATPD_STATE_CTIO;
1274                                 }
1275                         } else {
1276                                 atp->state = ATPD_STATE_CTIO;
1277                         }
1278                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: CTIO7[0x%x] seq %u nc %d CDB0=%x sstatus=0x%x flags=0x%x xfrlen=%u off=%u", __func__,
1279                             cto->ct_rxid, ATPD_GET_SEQNO(cto), ATPD_GET_NCAM(cto), atp->cdb0, cto->ct_scsi_status, cto->ct_flags, xfrlen, atp->bytes_xfered);
1280                 }
1281
1282                 if (isp_get_pcmd(isp, ccb)) {
1283                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "out of PCMDs\n");
1284                         TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe);
1285                         break;
1286                 }
1287                 handle = isp_allocate_handle(isp, ccb, ISP_HANDLE_TARGET);
1288                 if (handle == 0) {
1289                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "No XFLIST pointers for %s\n", __func__);
1290                         TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe);
1291                         isp_free_pcmd(isp, ccb);
1292                         break;
1293                 }
1294                 atp->bytes_in_transit += xfrlen;
1295                 PISP_PCMD(ccb)->datalen = xfrlen;
1296
1297                 /*
1298                  * Call the dma setup routines for this entry (and any subsequent
1299                  * CTIOs) if there's data to move, and then tell the f/w it's got
1300                  * new things to play with. As with isp_start's usage of DMA setup,
1301                  * any swizzling is done in the machine dependent layer. Because
1302                  * of this, we put the request onto the queue area first in native
1303                  * format.
1304                  */
1305                 cto->ct_syshandle = handle;
1306                 dmaresult = ISP_DMASETUP(isp, cso, cto);
1307                 if (dmaresult != 0) {
1308                         isp_destroy_handle(isp, handle);
1309                         isp_free_pcmd(isp, ccb);
1310                         if (dmaresult == CMD_EAGAIN) {
1311                                 TAILQ_INSERT_HEAD(waitq, &ccb->ccb_h, sim_links.tqe);
1312                                 break;
1313                         }
1314                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
1315                         xpt_done(ccb);
1316                         continue;
1317                 }
1318                 ccb->ccb_h.status = CAM_REQ_INPROG | CAM_SIM_QUEUED;
1319                 if (xfrlen) {
1320                         ccb->ccb_h.spriv_field0 = atp->bytes_xfered;
1321                 } else {
1322                         ccb->ccb_h.spriv_field0 = ~0;
1323                 }
1324                 atp->ctcnt++;
1325                 atp->seqno++;
1326         }
1327 }
1328
1329 static void
1330 isp_refire_notify_ack(void *arg)
1331 {
1332         isp_tna_t *tp  = arg;
1333         ispsoftc_t *isp = tp->isp;
1334
1335         ISP_ASSERT_LOCKED(isp);
1336         if (isp_notify_ack(isp, tp->not)) {
1337                 callout_schedule(&tp->timer, 5);
1338         } else {
1339                 free(tp, M_DEVBUF);
1340         }
1341 }
1342
1343
1344 static void
1345 isp_complete_ctio(ispsoftc_t *isp, union ccb *ccb)
1346 {
1347
1348         isp_rq_check_below(isp);
1349         ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1350         xpt_done(ccb);
1351 }
1352
1353 static void
1354 isp_handle_platform_atio7(ispsoftc_t *isp, at7_entry_t *aep)
1355 {
1356         int cdbxlen;
1357         lun_id_t lun;
1358         uint16_t chan, nphdl = NIL_HANDLE;
1359         uint32_t did, sid;
1360         fcportdb_t *lp;
1361         tstate_t *tptr;
1362         struct ccb_accept_tio *atiop;
1363         atio_private_data_t *atp = NULL;
1364         atio_private_data_t *oatp;
1365         inot_private_data_t *ntp;
1366
1367         did = (aep->at_hdr.d_id[0] << 16) | (aep->at_hdr.d_id[1] << 8) | aep->at_hdr.d_id[2];
1368         sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2];
1369         lun = CAM_EXTLUN_BYTE_SWIZZLE(be64dec(aep->at_cmnd.fcp_cmnd_lun));
1370
1371         if (ISP_CAP_MULTI_ID(isp) && isp->isp_nchan > 1) {
1372                 /* Channel has to be derived from D_ID */
1373                 isp_find_chan_by_did(isp, did, &chan);
1374                 if (chan == ISP_NOCHAN) {
1375                         isp_prt(isp, ISP_LOGWARN,
1376                             "%s: [RX_ID 0x%x] D_ID %x not found on any channel",
1377                             __func__, aep->at_rxid, did);
1378                         isp_endcmd(isp, aep, NIL_HANDLE, ISP_NOCHAN,
1379                             ECMD_TERMINATE, 0);
1380                         return;
1381                 }
1382         } else {
1383                 chan = 0;
1384         }
1385
1386         /*
1387          * Find the PDB entry for this initiator
1388          */
1389         if (isp_find_pdb_by_portid(isp, chan, sid, &lp) == 0) {
1390                 /*
1391                  * If we're not in the port database terminate the exchange.
1392                  */
1393                 isp_prt(isp, ISP_LOGTINFO, "%s: [RX_ID 0x%x] D_ID 0x%06x found on Chan %d for S_ID 0x%06x wasn't in PDB already",
1394                     __func__, aep->at_rxid, did, chan, sid);
1395                 isp_dump_portdb(isp, chan);
1396                 isp_endcmd(isp, aep, NIL_HANDLE, chan, ECMD_TERMINATE, 0);
1397                 return;
1398         }
1399         nphdl = lp->handle;
1400
1401         /*
1402          * Get the tstate pointer
1403          */
1404         tptr = get_lun_statep(isp, chan, lun);
1405         if (tptr == NULL) {
1406                 tptr = get_lun_statep(isp, chan, CAM_LUN_WILDCARD);
1407                 if (tptr == NULL) {
1408                         isp_prt(isp, ISP_LOGWARN,
1409                             "%s: [0x%x] no state pointer for lun %jx or wildcard",
1410                             __func__, aep->at_rxid, (uintmax_t)lun);
1411                         if (lun == 0) {
1412                                 isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_BUSY, 0);
1413                         } else {
1414                                 isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_CHECK_COND | ECMD_SVALID | (0x5 << 12) | (0x25 << 16), 0);
1415                         }
1416                         return;
1417                 }
1418         }
1419
1420         /*
1421          * Start any commands pending resources first.
1422          */
1423         if (isp_atio_restart(isp, chan, tptr))
1424                 goto noresrc;
1425
1426         /*
1427          * If the f/w is out of resources, just send a BUSY status back.
1428          */
1429         if (aep->at_rxid == AT7_NORESRC_RXID) {
1430                 isp_endcmd(isp, aep, nphdl, chan, SCSI_BUSY, 0);
1431                 return;
1432         }
1433
1434         /*
1435          * If we're out of resources, just send a BUSY status back.
1436          */
1437         atiop = (struct ccb_accept_tio *) SLIST_FIRST(&tptr->atios);
1438         if (atiop == NULL) {
1439                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atios", aep->at_rxid);
1440                 goto noresrc;
1441         }
1442
1443         oatp = isp_find_atpd(isp, chan, aep->at_rxid);
1444         if (oatp) {
1445                 isp_prt(isp, oatp->state == ATPD_STATE_LAST_CTIO ? ISP_LOGTDEBUG0 :
1446                     ISP_LOGWARN, "[0x%x] tag wraparound (N-Port Handle "
1447                     "0x%04x S_ID 0x%04x OX_ID 0x%04x) oatp state %d",
1448                     aep->at_rxid, nphdl, sid, aep->at_hdr.ox_id, oatp->state);
1449                 /*
1450                  * It's not a "no resource" condition- but we can treat it like one
1451                  */
1452                 goto noresrc;
1453         }
1454         atp = isp_get_atpd(isp, chan, aep->at_rxid);
1455         if (atp == NULL) {
1456                 isp_prt(isp, ISP_LOGTDEBUG0, "[0x%x] out of atps", aep->at_rxid);
1457                 isp_endcmd(isp, aep, nphdl, chan, SCSI_BUSY, 0);
1458                 return;
1459         }
1460         atp->word3 = lp->prli_word3;
1461         atp->state = ATPD_STATE_ATIO;
1462         SLIST_REMOVE_HEAD(&tptr->atios, sim_links.sle);
1463         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, atiop->ccb_h.path, "Take FREE ATIO\n");
1464         atiop->init_id = FC_PORTDB_TGT(isp, chan, lp);
1465         atiop->ccb_h.target_id = ISP_MAX_TARGETS(isp);
1466         atiop->ccb_h.target_lun = lun;
1467         atiop->sense_len = 0;
1468         cdbxlen = aep->at_cmnd.fcp_cmnd_alen_datadir >> FCP_CMND_ADDTL_CDBLEN_SHIFT;
1469         if (cdbxlen) {
1470                 isp_prt(isp, ISP_LOGWARN, "additional CDBLEN ignored");
1471         }
1472         cdbxlen = sizeof (aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb);
1473         ISP_MEMCPY(atiop->cdb_io.cdb_bytes, aep->at_cmnd.cdb_dl.sf.fcp_cmnd_cdb, cdbxlen);
1474         atiop->cdb_len = cdbxlen;
1475         atiop->ccb_h.status = CAM_CDB_RECVD;
1476         atiop->tag_id = atp->tag;
1477         switch (aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK) {
1478         case FCP_CMND_TASK_ATTR_SIMPLE:
1479                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
1480                 atiop->tag_action = MSG_SIMPLE_TASK;
1481                 break;
1482         case FCP_CMND_TASK_ATTR_HEAD:
1483                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
1484                 atiop->tag_action = MSG_HEAD_OF_QUEUE_TASK;
1485                 break;
1486         case FCP_CMND_TASK_ATTR_ORDERED:
1487                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
1488                 atiop->tag_action = MSG_ORDERED_TASK;
1489                 break;
1490         case FCP_CMND_TASK_ATTR_ACA:
1491                 atiop->ccb_h.flags |= CAM_TAG_ACTION_VALID;
1492                 atiop->tag_action = MSG_ACA_TASK;
1493                 break;
1494         case FCP_CMND_TASK_ATTR_UNTAGGED:
1495         default:
1496                 atiop->tag_action = 0;
1497                 break;
1498         }
1499         atiop->priority = (aep->at_cmnd.fcp_cmnd_task_attribute &
1500             FCP_CMND_PRIO_MASK) >> FCP_CMND_PRIO_SHIFT;
1501         atp->orig_datalen = aep->at_cmnd.cdb_dl.sf.fcp_cmnd_dl;
1502         atp->bytes_xfered = 0;
1503         atp->lun = lun;
1504         atp->nphdl = nphdl;
1505         atp->sid = sid;
1506         atp->did = did;
1507         atp->oxid = aep->at_hdr.ox_id;
1508         atp->rxid = aep->at_hdr.rx_id;
1509         atp->cdb0 = atiop->cdb_io.cdb_bytes[0];
1510         atp->tattr = aep->at_cmnd.fcp_cmnd_task_attribute & FCP_CMND_TASK_ATTR_MASK;
1511         atp->state = ATPD_STATE_CAM;
1512         isp_prt(isp, ISP_LOGTDEBUG0, "ATIO7[0x%x] CDB=0x%x lun %jx datalen %u",
1513             aep->at_rxid, atp->cdb0, (uintmax_t)lun, atp->orig_datalen);
1514         xpt_done((union ccb *)atiop);
1515         return;
1516 noresrc:
1517         KASSERT(atp == NULL, ("%s: atp is not NULL on noresrc!\n", __func__));
1518         ntp = isp_get_ntpd(isp, chan);
1519         if (ntp == NULL) {
1520                 isp_endcmd(isp, aep, nphdl, chan, SCSI_STATUS_BUSY, 0);
1521                 return;
1522         }
1523         memcpy(ntp->data, aep, QENTRY_LEN);
1524         STAILQ_INSERT_TAIL(&tptr->restart_queue, ntp, next);
1525 }
1526
1527
1528 /*
1529  * Handle starting an SRR (sequence retransmit request)
1530  * We get here when we've gotten the immediate notify
1531  * and the return of all outstanding CTIOs for this
1532  * transaction.
1533  */
1534 static void
1535 isp_handle_srr_start(ispsoftc_t *isp, atio_private_data_t *atp)
1536 {
1537         in_fcentry_24xx_t *inot;
1538         uint32_t srr_off, ccb_off, ccb_len, ccb_end;
1539         union ccb *ccb;
1540
1541         inot = (in_fcentry_24xx_t *)atp->srr;
1542         srr_off = inot->in_srr_reloff_lo | (inot->in_srr_reloff_hi << 16);
1543         ccb = atp->srr_ccb;
1544         atp->srr_ccb = NULL;
1545         atp->nsrr++;
1546         if (ccb == NULL) {
1547                 isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] null ccb", atp->tag);
1548                 goto fail;
1549         }
1550
1551         ccb_off = ccb->ccb_h.spriv_field0;
1552         ccb_len = ccb->csio.dxfer_len;
1553         ccb_end = (ccb_off == ~0)? ~0 : ccb_off + ccb_len;
1554
1555         switch (inot->in_srr_iu) {
1556         case R_CTL_INFO_SOLICITED_DATA:
1557                 /*
1558                  * We have to restart a FCP_DATA data out transaction
1559                  */
1560                 atp->sendst = 0;
1561                 atp->bytes_xfered = srr_off;
1562                 if (ccb_len == 0) {
1563                         isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x but current CCB doesn't transfer data", atp->tag, srr_off);
1564                         goto mdp;
1565                 }
1566                 if (srr_off < ccb_off || ccb_off > srr_off + ccb_len) {
1567                         isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x not covered by current CCB data range [0x%x..0x%x]", atp->tag, srr_off, ccb_off, ccb_end);
1568                         goto mdp;
1569                 }
1570                 isp_prt(isp, ISP_LOGWARN, "SRR[0x%x] SRR offset 0x%x covered by current CCB data range [0x%x..0x%x]", atp->tag, srr_off, ccb_off, ccb_end);
1571                 break;
1572         case R_CTL_INFO_COMMAND_STATUS:
1573                 isp_prt(isp, ISP_LOGTINFO, "SRR[0x%x] Got an FCP RSP SRR- resending status", atp->tag);
1574                 atp->sendst = 1;
1575                 /*
1576                  * We have to restart a FCP_RSP IU transaction
1577                  */
1578                 break;
1579         case R_CTL_INFO_DATA_DESCRIPTOR:
1580                 /*
1581                  * We have to restart an FCP DATA in transaction
1582                  */
1583                 isp_prt(isp, ISP_LOGWARN, "Got an FCP DATA IN SRR- dropping");
1584                 goto fail;
1585
1586         default:
1587                 isp_prt(isp, ISP_LOGWARN, "Got an unknown information (%x) SRR- dropping", inot->in_srr_iu);
1588                 goto fail;
1589         }
1590
1591         /*
1592          * We can't do anything until this is acked, so we might as well start it now.
1593          * We aren't going to do the usual asynchronous ack issue because we need
1594          * to make sure this gets on the wire first.
1595          */
1596         if (isp_notify_ack(isp, inot)) {
1597                 isp_prt(isp, ISP_LOGWARN, "could not push positive ack for SRR- you lose");
1598                 goto fail;
1599         }
1600         isp_target_start_ctio(isp, ccb, FROM_SRR);
1601         return;
1602 fail:
1603         inot->in_reserved = 1;
1604         isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
1605         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1606         ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
1607         isp_complete_ctio(isp, ccb);
1608         return;
1609 mdp:
1610         if (isp_notify_ack(isp, inot)) {
1611                 isp_prt(isp, ISP_LOGWARN, "could not push positive ack for SRR- you lose");
1612                 goto fail;
1613         }
1614         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1615         ccb->ccb_h.status |= CAM_MESSAGE_RECV;
1616         /*
1617          * This is not a strict interpretation of MDP, but it's close
1618          */
1619         ccb->csio.msg_ptr = &ccb->csio.sense_data.sense_buf[SSD_FULL_SIZE - 16];
1620         ccb->csio.msg_len = 7;
1621         ccb->csio.msg_ptr[0] = MSG_EXTENDED;
1622         ccb->csio.msg_ptr[1] = 5;
1623         ccb->csio.msg_ptr[2] = 0;       /* modify data pointer */
1624         ccb->csio.msg_ptr[3] = srr_off >> 24;
1625         ccb->csio.msg_ptr[4] = srr_off >> 16;
1626         ccb->csio.msg_ptr[5] = srr_off >> 8;
1627         ccb->csio.msg_ptr[6] = srr_off;
1628         isp_complete_ctio(isp, ccb);
1629 }
1630
1631
1632 static void
1633 isp_handle_platform_srr(ispsoftc_t *isp, isp_notify_t *notify)
1634 {
1635         in_fcentry_24xx_t *inot = notify->nt_lreserved;
1636         atio_private_data_t *atp;
1637         uint32_t tag = notify->nt_tagval & 0xffffffff;
1638
1639         atp = isp_find_atpd(isp, notify->nt_channel, tag);
1640         if (atp == NULL) {
1641                 isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x in SRR Notify",
1642                     __func__, tag);
1643                 isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, inot);
1644                 return;
1645         }
1646         atp->srr_notify_rcvd = 1;
1647         memcpy(atp->srr, inot, sizeof (atp->srr));
1648         isp_prt(isp, ISP_LOGTINFO, "SRR[0x%x] flags 0x%x srr_iu %x reloff 0x%x",
1649             inot->in_rxid, inot->in_flags, inot->in_srr_iu,
1650             ((uint32_t)inot->in_srr_reloff_hi << 16) | inot->in_srr_reloff_lo);
1651         if (atp->srr_ccb)
1652                 isp_handle_srr_start(isp, atp);
1653 }
1654
1655 static void
1656 isp_handle_platform_ctio(ispsoftc_t *isp, ct7_entry_t *ct)
1657 {
1658         union ccb *ccb;
1659         int sentstatus = 0, ok = 0, notify_cam = 0, failure = 0;
1660         atio_private_data_t *atp = NULL;
1661         int bus;
1662         uint32_t handle, data_requested, resid;
1663
1664         handle = ct->ct_syshandle;
1665         ccb = isp_find_xs(isp, handle);
1666         if (ccb == NULL) {
1667                 isp_print_bytes(isp, "null ccb in isp_handle_platform_ctio", QENTRY_LEN, ct);
1668                 return;
1669         }
1670         isp_destroy_handle(isp, handle);
1671         resid = data_requested = PISP_PCMD(ccb)->datalen;
1672         isp_free_pcmd(isp, ccb);
1673
1674         bus = XS_CHANNEL(ccb);
1675         atp = isp_find_atpd(isp, bus, ct->ct_rxid);
1676         if (atp == NULL) {
1677                 /*
1678                  * XXX: isp_clear_commands() generates fake CTIO with zero
1679                  * ct_rxid value, filling only ct_syshandle.  Workaround
1680                  * that using tag_id from the CCB, pointed by ct_syshandle.
1681                  */
1682                 atp = isp_find_atpd(isp, bus, ccb->csio.tag_id);
1683         }
1684         if (atp == NULL) {
1685                 isp_prt(isp, ISP_LOGERR, "%s: cannot find adjunct for %x after I/O", __func__, ccb->csio.tag_id);
1686                 return;
1687         }
1688         KASSERT((atp->ctcnt > 0), ("ctio count not greater than zero"));
1689         atp->bytes_in_transit -= data_requested;
1690         atp->ctcnt -= 1;
1691         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1692
1693         if (ct->ct_nphdl == CT7_SRR) {
1694                 atp->srr_ccb = ccb;
1695                 if (atp->srr_notify_rcvd)
1696                         isp_handle_srr_start(isp, atp);
1697                 return;
1698         }
1699         if (ct->ct_nphdl == CT_HBA_RESET) {
1700                 sentstatus = (ccb->ccb_h.flags & CAM_SEND_STATUS) &&
1701                     (atp->sendst == 0);
1702                 failure = CAM_UNREC_HBA_ERROR;
1703         } else {
1704                 sentstatus = ct->ct_flags & CT7_SENDSTATUS;
1705                 ok = (ct->ct_nphdl == CT7_OK);
1706                 notify_cam = (ct->ct_header.rqs_seqno & ATPD_SEQ_NOTIFY_CAM) != 0;
1707                 if ((ct->ct_flags & CT7_DATAMASK) != CT7_NO_DATA)
1708                         resid = ct->ct_resid;
1709         }
1710         isp_prt(isp, ok? ISP_LOGTDEBUG0 : ISP_LOGWARN, "%s: CTIO7[%x] seq %u nc %d sts 0x%x flg 0x%x sns %d resid %d %s", __func__, ct->ct_rxid, ATPD_GET_SEQNO(ct),
1711            notify_cam, ct->ct_nphdl, ct->ct_flags, (ccb->ccb_h.status & CAM_SENT_SENSE) != 0, resid, sentstatus? "FIN" : "MID");
1712         if (ok) {
1713                 if (data_requested > 0) {
1714                         atp->bytes_xfered += data_requested - resid;
1715                         ccb->csio.resid = ccb->csio.dxfer_len -
1716                             (data_requested - resid);
1717                 }
1718                 if (sentstatus && (ccb->ccb_h.flags & CAM_SEND_SENSE))
1719                         ccb->ccb_h.status |= CAM_SENT_SENSE;
1720                 ccb->ccb_h.status |= CAM_REQ_CMP;
1721         } else {
1722                 notify_cam = 1;
1723                 if (failure == CAM_UNREC_HBA_ERROR)
1724                         ccb->ccb_h.status |= CAM_UNREC_HBA_ERROR;
1725                 else
1726                         ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
1727         }
1728         atp->state = ATPD_STATE_PDON;
1729
1730         /*
1731          * We never *not* notify CAM when there has been any error (ok == 0),
1732          * so we never need to do an ATIO putback if we're not notifying CAM.
1733          */
1734         isp_prt(isp, ISP_LOGTDEBUG0, "%s CTIO[0x%x] done (ok=%d nc=%d nowsendstatus=%d ccb ss=%d)",
1735             (sentstatus)? "  FINAL " : "MIDTERM ", atp->tag, ok, notify_cam, atp->sendst, (ccb->ccb_h.flags & CAM_SEND_STATUS) != 0);
1736         if (notify_cam == 0) {
1737                 if (atp->sendst) {
1738                         isp_target_start_ctio(isp, ccb, FROM_CTIO_DONE);
1739                 }
1740                 return;
1741         }
1742
1743         /*
1744          * We are done with this ATIO if we successfully sent status.
1745          * In all other cases expect either another CTIO or XPT_ABORT.
1746          */
1747         if (ok && sentstatus)
1748                 isp_put_atpd(isp, bus, atp);
1749
1750         /*
1751          * We're telling CAM we're done with this CTIO transaction.
1752          *
1753          * 24XX cards never need an ATIO put back.
1754          */
1755         isp_complete_ctio(isp, ccb);
1756 }
1757
1758 static int
1759 isp_handle_platform_target_notify_ack(ispsoftc_t *isp, isp_notify_t *mp, uint32_t rsp)
1760 {
1761         ct7_entry_t local, *cto = &local;
1762
1763         if (isp->isp_state != ISP_RUNSTATE) {
1764                 isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) acked- h/w not ready (dropping)", mp->nt_ncode, mp->nt_lreserved != NULL);
1765                 return (0);
1766         }
1767
1768         /*
1769          * This case is for a Task Management Function, which shows up as an ATIO7 entry.
1770          */
1771         if (mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ATIO) {
1772                 at7_entry_t *aep = (at7_entry_t *)mp->nt_lreserved;
1773                 fcportdb_t *lp;
1774                 uint32_t sid;
1775                 uint16_t nphdl;
1776
1777                 sid = (aep->at_hdr.s_id[0] << 16) | (aep->at_hdr.s_id[1] << 8) | aep->at_hdr.s_id[2];
1778                 if (isp_find_pdb_by_portid(isp, mp->nt_channel, sid, &lp)) {
1779                         nphdl = lp->handle;
1780                 } else {
1781                         nphdl = NIL_HANDLE;
1782                 }
1783                 ISP_MEMZERO(cto, sizeof (ct7_entry_t));
1784                 cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
1785                 cto->ct_header.rqs_entry_count = 1;
1786                 cto->ct_nphdl = nphdl;
1787                 cto->ct_rxid = aep->at_rxid;
1788                 cto->ct_vpidx = mp->nt_channel;
1789                 cto->ct_iid_lo = sid;
1790                 cto->ct_iid_hi = sid >> 16;
1791                 cto->ct_oxid = aep->at_hdr.ox_id;
1792                 cto->ct_flags = CT7_SENDSTATUS|CT7_NOACK|CT7_NO_DATA|CT7_FLAG_MODE1;
1793                 cto->ct_flags |= (aep->at_ta_len >> 12) << CT7_TASK_ATTR_SHIFT;
1794                 if (rsp != 0) {
1795                         cto->ct_scsi_status |= (FCP_RSPLEN_VALID << 8);
1796                         cto->rsp.m1.ct_resplen = 4;
1797                         ISP_MEMZERO(cto->rsp.m1.ct_resp, sizeof (cto->rsp.m1.ct_resp));
1798                         cto->rsp.m1.ct_resp[0] = rsp & 0xff;
1799                         cto->rsp.m1.ct_resp[1] = (rsp >> 8) & 0xff;
1800                         cto->rsp.m1.ct_resp[2] = (rsp >> 16) & 0xff;
1801                         cto->rsp.m1.ct_resp[3] = (rsp >> 24) & 0xff;
1802                 }
1803                 return (isp_target_put_entry(isp, &cto));
1804         }
1805
1806         /*
1807          * This case is for a responding to an ABTS frame
1808          */
1809         if (mp->nt_lreserved && ((isphdr_t *)mp->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) {
1810
1811                 /*
1812                  * Overload nt_need_ack here to mark whether we've terminated the associated command.
1813                  */
1814                 if (mp->nt_need_ack) {
1815                         abts_t *abts = (abts_t *)mp->nt_lreserved;
1816
1817                         ISP_MEMZERO(cto, sizeof (ct7_entry_t));
1818                         isp_prt(isp, ISP_LOGTDEBUG0, "%s: [%x] terminating after ABTS received", __func__, abts->abts_rxid_task);
1819                         cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
1820                         cto->ct_header.rqs_entry_count = 1;
1821                         cto->ct_nphdl = mp->nt_nphdl;
1822                         cto->ct_rxid = abts->abts_rxid_task;
1823                         cto->ct_iid_lo = mp->nt_sid;
1824                         cto->ct_iid_hi = mp->nt_sid >> 16;
1825                         cto->ct_oxid = abts->abts_ox_id;
1826                         cto->ct_vpidx = mp->nt_channel;
1827                         cto->ct_flags = CT7_NOACK|CT7_TERMINATE;
1828                         if (isp_target_put_entry(isp, cto)) {
1829                                 return (ENOMEM);
1830                         }
1831                         mp->nt_need_ack = 0;
1832                 }
1833                 if (isp_acknak_abts(isp, mp->nt_lreserved, 0) == ENOMEM) {
1834                         return (ENOMEM);
1835                 } else {
1836                         return (0);
1837                 }
1838         }
1839
1840         /*
1841          * Handle logout cases here
1842          */
1843         if (mp->nt_ncode == NT_GLOBAL_LOGOUT) {
1844                 isp_del_all_wwn_entries(isp, mp->nt_channel);
1845         }
1846
1847         if (mp->nt_ncode == NT_LOGOUT)
1848                 isp_del_wwn_entries(isp, mp);
1849
1850         /*
1851          * General purpose acknowledgement
1852          */
1853         if (mp->nt_need_ack) {
1854                 isp_prt(isp, ISP_LOGTINFO, "Notify Code 0x%x (qevalid=%d) being acked", mp->nt_ncode, mp->nt_lreserved != NULL);
1855                 /*
1856                  * Don't need to use the guaranteed send because the caller can retry
1857                  */
1858                 return (isp_notify_ack(isp, mp->nt_lreserved));
1859         }
1860         return (0);
1861 }
1862
1863 /*
1864  * Handle task management functions.
1865  *
1866  * We show up here with a notify structure filled out.
1867  *
1868  * The nt_lreserved tag points to the original queue entry
1869  */
1870 static void
1871 isp_handle_platform_target_tmf(ispsoftc_t *isp, isp_notify_t *notify)
1872 {
1873         tstate_t *tptr;
1874         fcportdb_t *lp;
1875         struct ccb_immediate_notify *inot;
1876         inot_private_data_t *ntp = NULL;
1877         atio_private_data_t *atp;
1878         lun_id_t lun;
1879
1880         isp_prt(isp, ISP_LOGTDEBUG0, "%s: code 0x%x sid  0x%x tagval 0x%016llx chan %d lun %jx", __func__, notify->nt_ncode,
1881             notify->nt_sid, (unsigned long long) notify->nt_tagval, notify->nt_channel, notify->nt_lun);
1882         if (notify->nt_lun == LUN_ANY) {
1883                 if (notify->nt_tagval == TAG_ANY) {
1884                         lun = CAM_LUN_WILDCARD;
1885                 } else {
1886                         atp = isp_find_atpd(isp, notify->nt_channel,
1887                             notify->nt_tagval & 0xffffffff);
1888                         lun = atp ? atp->lun : CAM_LUN_WILDCARD;
1889                 }
1890         } else {
1891                 lun = notify->nt_lun;
1892         }
1893         tptr = get_lun_statep(isp, notify->nt_channel, lun);
1894         if (tptr == NULL) {
1895                 tptr = get_lun_statep(isp, notify->nt_channel, CAM_LUN_WILDCARD);
1896                 if (tptr == NULL) {
1897                         isp_prt(isp, ISP_LOGWARN, "%s: no state pointer found for chan %d lun %#jx", __func__, notify->nt_channel, (uintmax_t)lun);
1898                         goto bad;
1899                 }
1900         }
1901         inot = (struct ccb_immediate_notify *) SLIST_FIRST(&tptr->inots);
1902         if (inot == NULL) {
1903                 isp_prt(isp, ISP_LOGWARN, "%s: out of immediate notify structures for chan %d lun %#jx", __func__, notify->nt_channel, (uintmax_t)lun);
1904                 goto bad;
1905         }
1906
1907         inot->ccb_h.target_id = ISP_MAX_TARGETS(isp);
1908         inot->ccb_h.target_lun = lun;
1909         if (isp_find_pdb_by_portid(isp, notify->nt_channel, notify->nt_sid, &lp) == 0 &&
1910             isp_find_pdb_by_handle(isp, notify->nt_channel, notify->nt_nphdl, &lp) == 0) {
1911                 inot->initiator_id = CAM_TARGET_WILDCARD;
1912         } else {
1913                 inot->initiator_id = FC_PORTDB_TGT(isp, notify->nt_channel, lp);
1914         }
1915         inot->seq_id = notify->nt_tagval;
1916         inot->tag_id = notify->nt_tagval >> 32;
1917
1918         switch (notify->nt_ncode) {
1919         case NT_ABORT_TASK:
1920                 isp_target_mark_aborted_early(isp, notify->nt_channel, tptr, inot->tag_id);
1921                 inot->arg = MSG_ABORT_TASK;
1922                 break;
1923         case NT_ABORT_TASK_SET:
1924                 isp_target_mark_aborted_early(isp, notify->nt_channel, tptr, TAG_ANY);
1925                 inot->arg = MSG_ABORT_TASK_SET;
1926                 break;
1927         case NT_CLEAR_ACA:
1928                 inot->arg = MSG_CLEAR_ACA;
1929                 break;
1930         case NT_CLEAR_TASK_SET:
1931                 inot->arg = MSG_CLEAR_TASK_SET;
1932                 break;
1933         case NT_LUN_RESET:
1934                 inot->arg = MSG_LOGICAL_UNIT_RESET;
1935                 break;
1936         case NT_TARGET_RESET:
1937                 inot->arg = MSG_TARGET_RESET;
1938                 break;
1939         case NT_QUERY_TASK_SET:
1940                 inot->arg = MSG_QUERY_TASK_SET;
1941                 break;
1942         case NT_QUERY_ASYNC_EVENT:
1943                 inot->arg = MSG_QUERY_ASYNC_EVENT;
1944                 break;
1945         default:
1946                 isp_prt(isp, ISP_LOGWARN, "%s: unknown TMF code 0x%x for chan %d lun %#jx", __func__, notify->nt_ncode, notify->nt_channel, (uintmax_t)lun);
1947                 goto bad;
1948         }
1949
1950         ntp = isp_get_ntpd(isp, notify->nt_channel);
1951         if (ntp == NULL) {
1952                 isp_prt(isp, ISP_LOGWARN, "%s: out of inotify private structures", __func__);
1953                 goto bad;
1954         }
1955         ISP_MEMCPY(&ntp->nt, notify, sizeof (isp_notify_t));
1956         if (notify->nt_lreserved) {
1957                 ISP_MEMCPY(&ntp->data, notify->nt_lreserved, QENTRY_LEN);
1958                 ntp->nt.nt_lreserved = &ntp->data;
1959         }
1960         ntp->seq_id = notify->nt_tagval;
1961         ntp->tag_id = notify->nt_tagval >> 32;
1962
1963         SLIST_REMOVE_HEAD(&tptr->inots, sim_links.sle);
1964         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, inot->ccb_h.path, "Take FREE INOT\n");
1965         inot->ccb_h.status = CAM_MESSAGE_RECV;
1966         xpt_done((union ccb *)inot);
1967         return;
1968 bad:
1969         if (notify->nt_need_ack) {
1970                 if (((isphdr_t *)notify->nt_lreserved)->rqs_entry_type == RQSTYPE_ABTS_RCVD) {
1971                         if (isp_acknak_abts(isp, notify->nt_lreserved, ENOMEM)) {
1972                                 isp_prt(isp, ISP_LOGWARN, "you lose- unable to send an ACKNAK");
1973                         }
1974                 } else {
1975                         isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK, notify->nt_lreserved);
1976                 }
1977         }
1978 }
1979
1980 static void
1981 isp_target_mark_aborted_early(ispsoftc_t *isp, int chan, tstate_t *tptr, uint32_t tag_id)
1982 {
1983         atio_private_data_t *atp, *atpool;
1984         inot_private_data_t *ntp, *tmp;
1985         uint32_t this_tag_id;
1986
1987         /*
1988          * First, clean any commands pending restart
1989          */
1990         STAILQ_FOREACH_SAFE(ntp, &tptr->restart_queue, next, tmp) {
1991                 this_tag_id = ((at7_entry_t *)ntp->data)->at_rxid;
1992                 if ((uint64_t)tag_id == TAG_ANY || tag_id == this_tag_id) {
1993                         isp_endcmd(isp, ntp->data, NIL_HANDLE, chan,
1994                             ECMD_TERMINATE, 0);
1995                         isp_put_ntpd(isp, chan, ntp);
1996                         STAILQ_REMOVE(&tptr->restart_queue, ntp,
1997                             inot_private_data, next);
1998                 }
1999         }
2000
2001         /*
2002          * Now mark other ones dead as well.
2003          */
2004         ISP_GET_PC(isp, chan, atpool, atpool);
2005         for (atp = atpool; atp < &atpool[ATPDPSIZE]; atp++) {
2006                 if (atp->lun != tptr->ts_lun)
2007                         continue;
2008                 if ((uint64_t)tag_id == TAG_ANY || atp->tag == tag_id)
2009                         atp->dead = 1;
2010         }
2011 }
2012 #endif
2013
2014 static void
2015 isp_poll(struct cam_sim *sim)
2016 {
2017         ispsoftc_t *isp = cam_sim_softc(sim);
2018
2019         ISP_RUN_ISR(isp);
2020 }
2021
2022
2023 static void
2024 isp_watchdog(void *arg)
2025 {
2026         struct ccb_scsiio *xs = arg;
2027         ispsoftc_t *isp;
2028         uint32_t ohandle = ISP_HANDLE_FREE, handle;
2029
2030         isp = XS_ISP(xs);
2031
2032         handle = isp_find_handle(isp, xs);
2033
2034         /*
2035          * Hand crank the interrupt code just to be sure the command isn't stuck somewhere.
2036          */
2037         if (handle != ISP_HANDLE_FREE) {
2038                 ISP_RUN_ISR(isp);
2039                 ohandle = handle;
2040                 handle = isp_find_handle(isp, xs);
2041         }
2042         if (handle != ISP_HANDLE_FREE) {
2043                 /*
2044                  * Try and make sure the command is really dead before
2045                  * we release the handle (and DMA resources) for reuse.
2046                  *
2047                  * If we are successful in aborting the command then
2048                  * we're done here because we'll get the command returned
2049                  * back separately.
2050                  */
2051                 if (isp_control(isp, ISPCTL_ABORT_CMD, xs) == 0) {
2052                         return;
2053                 }
2054
2055                 /*
2056                  * Note that after calling the above, the command may in
2057                  * fact have been completed.
2058                  */
2059                 xs = isp_find_xs(isp, handle);
2060
2061                 /*
2062                  * If the command no longer exists, then we won't
2063                  * be able to find the xs again with this handle.
2064                  */
2065                 if (xs == NULL) {
2066                         return;
2067                 }
2068
2069                 /*
2070                  * After this point, the command is really dead.
2071                  */
2072                 ISP_DMAFREE(isp, xs);
2073                 isp_destroy_handle(isp, handle);
2074                 isp_prt(isp, ISP_LOGERR, "%s: timeout for handle 0x%x", __func__, handle);
2075                 XS_SETERR(xs, CAM_CMD_TIMEOUT);
2076                 isp_done(xs);
2077         } else {
2078                 if (ohandle != ISP_HANDLE_FREE) {
2079                         isp_prt(isp, ISP_LOGWARN, "%s: timeout for handle 0x%x, recovered during interrupt", __func__, ohandle);
2080                 } else {
2081                         isp_prt(isp, ISP_LOGWARN, "%s: timeout for handle already free", __func__);
2082                 }
2083         }
2084 }
2085
2086 static void
2087 isp_make_here(ispsoftc_t *isp, fcportdb_t *fcp, int chan, int tgt)
2088 {
2089         union ccb *ccb;
2090         struct isp_fc *fc = ISP_FC_PC(isp, chan);
2091
2092         /*
2093          * Allocate a CCB, create a wildcard path for this target and schedule a rescan.
2094          */
2095         ccb = xpt_alloc_ccb_nowait();
2096         if (ccb == NULL) {
2097                 isp_prt(isp, ISP_LOGWARN, "Chan %d unable to alloc CCB for rescan", chan);
2098                 return;
2099         }
2100         if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(fc->sim),
2101             tgt, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
2102                 isp_prt(isp, ISP_LOGWARN, "unable to create path for rescan");
2103                 xpt_free_ccb(ccb);
2104                 return;
2105         }
2106         xpt_rescan(ccb);
2107 }
2108
2109 static void
2110 isp_make_gone(ispsoftc_t *isp, fcportdb_t *fcp, int chan, int tgt)
2111 {
2112         struct cam_path *tp;
2113         struct isp_fc *fc = ISP_FC_PC(isp, chan);
2114
2115         if (xpt_create_path(&tp, NULL, cam_sim_path(fc->sim), tgt, CAM_LUN_WILDCARD) == CAM_REQ_CMP) {
2116                 xpt_async(AC_LOST_DEVICE, tp, NULL);
2117                 xpt_free_path(tp);
2118         }
2119 }
2120
2121 /*
2122  * Gone Device Timer Function- when we have decided that a device has gone
2123  * away, we wait a specific period of time prior to telling the OS it has
2124  * gone away.
2125  *
2126  * This timer function fires once a second and then scans the port database
2127  * for devices that are marked dead but still have a virtual target assigned.
2128  * We decrement a counter for that port database entry, and when it hits zero,
2129  * we tell the OS the device has gone away.
2130  */
2131 static void
2132 isp_gdt(void *arg)
2133 {
2134         struct isp_fc *fc = arg;
2135         taskqueue_enqueue(taskqueue_thread, &fc->gtask);
2136 }
2137
2138 static void
2139 isp_gdt_task(void *arg, int pending)
2140 {
2141         struct isp_fc *fc = arg;
2142         ispsoftc_t *isp = fc->isp;
2143         int chan = fc - isp->isp_osinfo.pc.fc;
2144         fcportdb_t *lp;
2145         struct ac_contract ac;
2146         struct ac_device_changed *adc;
2147         int dbidx, more_to_do = 0;
2148
2149         ISP_LOCK(isp);
2150         isp_prt(isp, ISP_LOGDEBUG0, "Chan %d GDT timer expired", chan);
2151         for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
2152                 lp = &FCPARAM(isp, chan)->portdb[dbidx];
2153
2154                 if (lp->state != FC_PORTDB_STATE_ZOMBIE) {
2155                         continue;
2156                 }
2157                 if (lp->gone_timer != 0) {
2158                         lp->gone_timer -= 1;
2159                         more_to_do++;
2160                         continue;
2161                 }
2162                 isp_prt(isp, ISP_LOGCONFIG, prom3, chan, dbidx, lp->portid, "Gone Device Timeout");
2163                 if (lp->is_target) {
2164                         lp->is_target = 0;
2165                         isp_make_gone(isp, lp, chan, dbidx);
2166                 }
2167                 if (lp->is_initiator) {
2168                         lp->is_initiator = 0;
2169                         ac.contract_number = AC_CONTRACT_DEV_CHG;
2170                         adc = (struct ac_device_changed *) ac.contract_data;
2171                         adc->wwpn = lp->port_wwn;
2172                         adc->port = lp->portid;
2173                         adc->target = dbidx;
2174                         adc->arrived = 0;
2175                         xpt_async(AC_CONTRACT, fc->path, &ac);
2176                 }
2177                 lp->state = FC_PORTDB_STATE_NIL;
2178         }
2179         if (fc->ready) {
2180                 if (more_to_do) {
2181                         callout_reset(&fc->gdt, hz, isp_gdt, fc);
2182                 } else {
2183                         callout_deactivate(&fc->gdt);
2184                         isp_prt(isp, ISP_LOG_SANCFG, "Chan %d Stopping Gone Device Timer @ %lu", chan, (unsigned long) time_uptime);
2185                 }
2186         }
2187         ISP_UNLOCK(isp);
2188 }
2189
2190 /*
2191  * When loop goes down we remember the time and freeze CAM command queue.
2192  * During some time period we are trying to reprobe the loop.  But if we
2193  * fail, we tell the OS that devices have gone away and drop the freeze.
2194  *
2195  * We don't clear the devices out of our port database because, when loop
2196  * come back up, we have to do some actual cleanup with the chip at that
2197  * point (implicit PLOGO, e.g., to get the chip's port database state right).
2198  */
2199 static void
2200 isp_loop_changed(ispsoftc_t *isp, int chan)
2201 {
2202         fcparam *fcp = FCPARAM(isp, chan);
2203         struct isp_fc *fc = ISP_FC_PC(isp, chan);
2204
2205         if (fc->loop_down_time)
2206                 return;
2207         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Loop changed", chan);
2208         if (fcp->role & ISP_ROLE_INITIATOR)
2209                 isp_freeze_loopdown(isp, chan);
2210         fc->loop_down_time = time_uptime;
2211         wakeup(fc);
2212 }
2213
2214 static void
2215 isp_loop_up(ispsoftc_t *isp, int chan)
2216 {
2217         struct isp_fc *fc = ISP_FC_PC(isp, chan);
2218
2219         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Loop is up", chan);
2220         fc->loop_seen_once = 1;
2221         fc->loop_down_time = 0;
2222         isp_unfreeze_loopdown(isp, chan);
2223 }
2224
2225 static void
2226 isp_loop_dead(ispsoftc_t *isp, int chan)
2227 {
2228         fcparam *fcp = FCPARAM(isp, chan);
2229         struct isp_fc *fc = ISP_FC_PC(isp, chan);
2230         fcportdb_t *lp;
2231         struct ac_contract ac;
2232         struct ac_device_changed *adc;
2233         int dbidx, i;
2234
2235         isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Loop is dead", chan);
2236
2237         /*
2238          * Notify to the OS all targets who we now consider have departed.
2239          */
2240         for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
2241                 lp = &fcp->portdb[dbidx];
2242
2243                 if (lp->state == FC_PORTDB_STATE_NIL)
2244                         continue;
2245
2246                 for (i = 0; i < ISP_HANDLE_NUM(isp); i++) {
2247                         struct ccb_scsiio *xs;
2248
2249                         if (ISP_H2HT(isp->isp_xflist[i].handle) != ISP_HANDLE_INITIATOR) {
2250                                 continue;
2251                         }
2252                         if ((xs = isp->isp_xflist[i].cmd) == NULL) {
2253                                 continue;
2254                         }
2255                         if (dbidx != XS_TGT(xs)) {
2256                                 continue;
2257                         }
2258                         isp_prt(isp, ISP_LOGWARN, "command handle 0x%x for %d.%d.%jx orphaned by loop down timeout",
2259                             isp->isp_xflist[i].handle, chan, XS_TGT(xs),
2260                             (uintmax_t)XS_LUN(xs));
2261
2262                         /*
2263                          * Just like in isp_watchdog, abort the outstanding
2264                          * command or immediately free its resources if it is
2265                          * not active
2266                          */
2267                         if (isp_control(isp, ISPCTL_ABORT_CMD, xs) == 0) {
2268                                 continue;
2269                         }
2270
2271                         ISP_DMAFREE(isp, xs);
2272                         isp_destroy_handle(isp, isp->isp_xflist[i].handle);
2273                         isp_prt(isp, ISP_LOGWARN, "command handle 0x%x for %d.%d.%jx could not be aborted and was destroyed",
2274                             isp->isp_xflist[i].handle, chan, XS_TGT(xs),
2275                             (uintmax_t)XS_LUN(xs));
2276                         XS_SETERR(xs, HBA_BUSRESET);
2277                         isp_done(xs);
2278                 }
2279
2280                 isp_prt(isp, ISP_LOGCONFIG, prom3, chan, dbidx, lp->portid, "Loop Down Timeout");
2281                 if (lp->is_target) {
2282                         lp->is_target = 0;
2283                         isp_make_gone(isp, lp, chan, dbidx);
2284                 }
2285                 if (lp->is_initiator) {
2286                         lp->is_initiator = 0;
2287                         ac.contract_number = AC_CONTRACT_DEV_CHG;
2288                         adc = (struct ac_device_changed *) ac.contract_data;
2289                         adc->wwpn = lp->port_wwn;
2290                         adc->port = lp->portid;
2291                         adc->target = dbidx;
2292                         adc->arrived = 0;
2293                         xpt_async(AC_CONTRACT, fc->path, &ac);
2294                 }
2295         }
2296
2297         isp_unfreeze_loopdown(isp, chan);
2298         fc->loop_down_time = 0;
2299 }
2300
2301 static void
2302 isp_kthread(void *arg)
2303 {
2304         struct isp_fc *fc = arg;
2305         ispsoftc_t *isp = fc->isp;
2306         int chan = fc - isp->isp_osinfo.pc.fc;
2307         int slp = 0, d;
2308         int lb, lim;
2309
2310         ISP_LOCK(isp);
2311         while (isp->isp_osinfo.is_exiting == 0) {
2312                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0,
2313                     "Chan %d Checking FC state", chan);
2314                 lb = isp_fc_runstate(isp, chan, 250000);
2315                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0,
2316                     "Chan %d FC got to %s state", chan,
2317                     isp_fc_loop_statename(lb));
2318
2319                 /*
2320                  * Our action is different based upon whether we're supporting
2321                  * Initiator mode or not. If we are, we might freeze the simq
2322                  * when loop is down and set all sorts of different delays to
2323                  * check again.
2324                  *
2325                  * If not, we simply just wait for loop to come up.
2326                  */
2327                 if (lb == LOOP_READY || lb < 0) {
2328                         slp = 0;
2329                 } else {
2330                         /*
2331                          * If we've never seen loop up and we've waited longer
2332                          * than quickboot time, or we've seen loop up but we've
2333                          * waited longer than loop_down_limit, give up and go
2334                          * to sleep until loop comes up.
2335                          */
2336                         if (fc->loop_seen_once == 0)
2337                                 lim = isp_quickboot_time;
2338                         else
2339                                 lim = fc->loop_down_limit;
2340                         d = time_uptime - fc->loop_down_time;
2341                         if (d >= lim)
2342                                 slp = 0;
2343                         else if (d < 10)
2344                                 slp = 1;
2345                         else if (d < 30)
2346                                 slp = 5;
2347                         else if (d < 60)
2348                                 slp = 10;
2349                         else if (d < 120)
2350                                 slp = 20;
2351                         else
2352                                 slp = 30;
2353                 }
2354
2355                 if (slp == 0) {
2356                         if (lb == LOOP_READY)
2357                                 isp_loop_up(isp, chan);
2358                         else
2359                                 isp_loop_dead(isp, chan);
2360                 }
2361
2362                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0,
2363                     "Chan %d sleep for %d seconds", chan, slp);
2364                 msleep(fc, &isp->isp_lock, PRIBIO, "ispf", slp * hz);
2365         }
2366         fc->num_threads -= 1;
2367         ISP_UNLOCK(isp);
2368         kthread_exit();
2369 }
2370
2371 #ifdef  ISP_TARGET_MODE
2372 static void
2373 isp_abort_atio(ispsoftc_t *isp, union ccb *ccb)
2374 {
2375         atio_private_data_t *atp;
2376         union ccb *accb = ccb->cab.abort_ccb;
2377         struct ccb_hdr *sccb;
2378         tstate_t *tptr;
2379
2380         tptr = get_lun_statep(isp, XS_CHANNEL(accb), XS_LUN(accb));
2381         if (tptr != NULL) {
2382                 /* Search for the ATIO among queueued. */
2383                 SLIST_FOREACH(sccb, &tptr->atios, sim_links.sle) {
2384                         if (sccb != &accb->ccb_h)
2385                                 continue;
2386                         SLIST_REMOVE(&tptr->atios, sccb, ccb_hdr, sim_links.sle);
2387                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, sccb->path,
2388                             "Abort FREE ATIO\n");
2389                         accb->ccb_h.status = CAM_REQ_ABORTED;
2390                         xpt_done(accb);
2391                         ccb->ccb_h.status = CAM_REQ_CMP;
2392                         return;
2393                 }
2394         }
2395
2396         /* Search for the ATIO among running. */
2397         atp = isp_find_atpd(isp, XS_CHANNEL(accb), accb->atio.tag_id);
2398         if (atp != NULL) {
2399                 /* Send TERMINATE to firmware. */
2400                 if (!atp->dead) {
2401                         uint8_t storage[QENTRY_LEN];
2402                         ct7_entry_t *cto = (ct7_entry_t *) storage;
2403
2404                         ISP_MEMZERO(cto, sizeof (ct7_entry_t));
2405                         cto->ct_header.rqs_entry_type = RQSTYPE_CTIO7;
2406                         cto->ct_header.rqs_entry_count = 1;
2407                         cto->ct_nphdl = atp->nphdl;
2408                         cto->ct_rxid = atp->tag;
2409                         cto->ct_iid_lo = atp->sid;
2410                         cto->ct_iid_hi = atp->sid >> 16;
2411                         cto->ct_oxid = atp->oxid;
2412                         cto->ct_vpidx = XS_CHANNEL(accb);
2413                         cto->ct_flags = CT7_NOACK|CT7_TERMINATE;
2414                         isp_target_put_entry(isp, cto);
2415                 }
2416                 isp_put_atpd(isp, XS_CHANNEL(accb), atp);
2417                 ccb->ccb_h.status = CAM_REQ_CMP;
2418         } else {
2419                 ccb->ccb_h.status = CAM_UA_ABORT;
2420         }
2421 }
2422
2423 static void
2424 isp_abort_inot(ispsoftc_t *isp, union ccb *ccb)
2425 {
2426         inot_private_data_t *ntp;
2427         union ccb *accb = ccb->cab.abort_ccb;
2428         struct ccb_hdr *sccb;
2429         tstate_t *tptr;
2430
2431         tptr = get_lun_statep(isp, XS_CHANNEL(accb), XS_LUN(accb));
2432         if (tptr != NULL) {
2433                 /* Search for the INOT among queueued. */
2434                 SLIST_FOREACH(sccb, &tptr->inots, sim_links.sle) {
2435                         if (sccb != &accb->ccb_h)
2436                                 continue;
2437                         SLIST_REMOVE(&tptr->inots, sccb, ccb_hdr, sim_links.sle);
2438                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, sccb->path,
2439                             "Abort FREE INOT\n");
2440                         accb->ccb_h.status = CAM_REQ_ABORTED;
2441                         xpt_done(accb);
2442                         ccb->ccb_h.status = CAM_REQ_CMP;
2443                         return;
2444                 }
2445         }
2446
2447         /* Search for the INOT among running. */
2448         ntp = isp_find_ntpd(isp, XS_CHANNEL(accb), accb->cin1.tag_id, accb->cin1.seq_id);
2449         if (ntp != NULL) {
2450                 if (ntp->nt.nt_need_ack) {
2451                         isp_async(isp, ISPASYNC_TARGET_NOTIFY_ACK,
2452                             ntp->nt.nt_lreserved);
2453                 }
2454                 isp_put_ntpd(isp, XS_CHANNEL(accb), ntp);
2455                 ccb->ccb_h.status = CAM_REQ_CMP;
2456         } else {
2457                 ccb->ccb_h.status = CAM_UA_ABORT;
2458                 return;
2459         }
2460 }
2461 #endif
2462
2463 static void
2464 isp_action(struct cam_sim *sim, union ccb *ccb)
2465 {
2466         int bus, tgt, error;
2467         ispsoftc_t *isp;
2468         fcparam *fcp;
2469         struct ccb_trans_settings *cts;
2470         sbintime_t ts;
2471
2472         CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("isp_action\n"));
2473
2474         isp = (ispsoftc_t *)cam_sim_softc(sim);
2475         ISP_ASSERT_LOCKED(isp);
2476         bus = cam_sim_bus(sim);
2477         isp_prt(isp, ISP_LOGDEBUG2, "isp_action code %x", ccb->ccb_h.func_code);
2478         ISP_PCMD(ccb) = NULL;
2479
2480         switch (ccb->ccb_h.func_code) {
2481         case XPT_SCSI_IO:       /* Execute the requested I/O operation */
2482                 /*
2483                  * Do a couple of preliminary checks...
2484                  */
2485                 if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0) {
2486                         if ((ccb->ccb_h.flags & CAM_CDB_PHYS) != 0) {
2487                                 ccb->ccb_h.status = CAM_REQ_INVALID;
2488                                 isp_done((struct ccb_scsiio *) ccb);
2489                                 break;
2490                         }
2491                 }
2492 #ifdef  DIAGNOSTIC
2493                 if (ccb->ccb_h.target_id >= ISP_MAX_TARGETS(isp)) {
2494                         xpt_print(ccb->ccb_h.path, "invalid target\n");
2495                         ccb->ccb_h.status = CAM_PATH_INVALID;
2496                 }
2497                 if (ccb->ccb_h.status == CAM_PATH_INVALID) {
2498                         xpt_done(ccb);
2499                         break;
2500                 }
2501 #endif
2502                 ccb->csio.scsi_status = SCSI_STATUS_OK;
2503                 if (isp_get_pcmd(isp, ccb)) {
2504                         isp_prt(isp, ISP_LOGWARN, "out of PCMDs");
2505                         cam_freeze_devq(ccb->ccb_h.path);
2506                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 250, 0);
2507                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
2508                         xpt_done(ccb);
2509                         break;
2510                 }
2511                 error = isp_start((XS_T *) ccb);
2512                 isp_rq_check_above(isp);
2513                 switch (error) {
2514                 case 0:
2515                         ccb->ccb_h.status |= CAM_SIM_QUEUED;
2516                         if (ccb->ccb_h.timeout == CAM_TIME_INFINITY)
2517                                 break;
2518                         /* Give firmware extra 10s to handle timeout. */
2519                         ts = SBT_1MS * ccb->ccb_h.timeout + 10 * SBT_1S;
2520                         callout_reset_sbt(&PISP_PCMD(ccb)->wdog, ts, 0,
2521                             isp_watchdog, ccb, 0);
2522                         break;
2523                 case CMD_RQLATER:
2524                         isp_prt(isp, ISP_LOGDEBUG0, "%d.%jx retry later",
2525                             XS_TGT(ccb), (uintmax_t)XS_LUN(ccb));
2526                         cam_freeze_devq(ccb->ccb_h.path);
2527                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 1000, 0);
2528                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
2529                         isp_free_pcmd(isp, ccb);
2530                         xpt_done(ccb);
2531                         break;
2532                 case CMD_EAGAIN:
2533                         isp_free_pcmd(isp, ccb);
2534                         cam_freeze_devq(ccb->ccb_h.path);
2535                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 10, 0);
2536                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
2537                         xpt_done(ccb);
2538                         break;
2539                 case CMD_COMPLETE:
2540                         isp_done((struct ccb_scsiio *) ccb);
2541                         break;
2542                 default:
2543                         isp_prt(isp, ISP_LOGERR, "What's this? 0x%x at %d in file %s", error, __LINE__, __FILE__);
2544                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
2545                         isp_free_pcmd(isp, ccb);
2546                         xpt_done(ccb);
2547                 }
2548                 break;
2549
2550 #ifdef  ISP_TARGET_MODE
2551         case XPT_EN_LUN:                /* Enable/Disable LUN as a target */
2552                 if (ccb->cel.enable) {
2553                         isp_enable_lun(isp, ccb);
2554                 } else {
2555                         isp_disable_lun(isp, ccb);
2556                 }
2557                 break;
2558         case XPT_IMMEDIATE_NOTIFY:      /* Add Immediate Notify Resource */
2559         case XPT_ACCEPT_TARGET_IO:      /* Add Accept Target IO Resource */
2560         {
2561                 tstate_t *tptr = get_lun_statep(isp, XS_CHANNEL(ccb), ccb->ccb_h.target_lun);
2562                 if (tptr == NULL) {
2563                         const char *str;
2564
2565                         if (ccb->ccb_h.func_code == XPT_IMMEDIATE_NOTIFY)
2566                                 str = "XPT_IMMEDIATE_NOTIFY";
2567                         else
2568                                 str = "XPT_ACCEPT_TARGET_IO";
2569                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path,
2570                             "%s: no state pointer found for %s\n",
2571                             __func__, str);
2572                         ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2573                         xpt_done(ccb);
2574                         break;
2575                 }
2576
2577                 if (ccb->ccb_h.func_code == XPT_ACCEPT_TARGET_IO) {
2578                         ccb->atio.tag_id = 0;
2579                         SLIST_INSERT_HEAD(&tptr->atios, &ccb->ccb_h, sim_links.sle);
2580                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, ccb->ccb_h.path,
2581                             "Put FREE ATIO\n");
2582                 } else if (ccb->ccb_h.func_code == XPT_IMMEDIATE_NOTIFY) {
2583                         ccb->cin1.seq_id = ccb->cin1.tag_id = 0;
2584                         SLIST_INSERT_HEAD(&tptr->inots, &ccb->ccb_h, sim_links.sle);
2585                         ISP_PATH_PRT(isp, ISP_LOGTDEBUG2, ccb->ccb_h.path,
2586                             "Put FREE INOT\n");
2587                 }
2588                 ccb->ccb_h.status = CAM_REQ_INPROG;
2589                 break;
2590         }
2591         case XPT_NOTIFY_ACKNOWLEDGE:            /* notify ack */
2592         {
2593                 inot_private_data_t *ntp;
2594
2595                 /*
2596                  * XXX: Because we cannot guarantee that the path information in the notify acknowledge ccb
2597                  * XXX: matches that for the immediate notify, we have to *search* for the notify structure
2598                  */
2599                 /*
2600                  * All the relevant path information is in the associated immediate notify
2601                  */
2602                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: [0x%x] NOTIFY ACKNOWLEDGE for 0x%x seen\n", __func__, ccb->cna2.tag_id, ccb->cna2.seq_id);
2603                 ntp = isp_find_ntpd(isp, XS_CHANNEL(ccb), ccb->cna2.tag_id, ccb->cna2.seq_id);
2604                 if (ntp == NULL) {
2605                         ISP_PATH_PRT(isp, ISP_LOGWARN, ccb->ccb_h.path, "%s: [0x%x] XPT_NOTIFY_ACKNOWLEDGE of 0x%x cannot find ntp private data\n", __func__,
2606                              ccb->cna2.tag_id, ccb->cna2.seq_id);
2607                         ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2608                         xpt_done(ccb);
2609                         break;
2610                 }
2611                 if (isp_handle_platform_target_notify_ack(isp, &ntp->nt,
2612                     (ccb->ccb_h.flags & CAM_SEND_STATUS) ? ccb->cna2.arg : 0)) {
2613                         cam_freeze_devq(ccb->ccb_h.path);
2614                         cam_release_devq(ccb->ccb_h.path, RELSIM_RELEASE_AFTER_TIMEOUT, 0, 1000, 0);
2615                         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2616                         ccb->ccb_h.status |= CAM_REQUEUE_REQ;
2617                         break;
2618                 }
2619                 isp_put_ntpd(isp, XS_CHANNEL(ccb), ntp);
2620                 ccb->ccb_h.status = CAM_REQ_CMP;
2621                 ISP_PATH_PRT(isp, ISP_LOGTDEBUG0, ccb->ccb_h.path, "%s: [0x%x] calling xpt_done for tag 0x%x\n", __func__, ccb->cna2.tag_id, ccb->cna2.seq_id);
2622                 xpt_done(ccb);
2623                 break;
2624         }
2625         case XPT_CONT_TARGET_IO:
2626                 isp_target_start_ctio(isp, ccb, FROM_CAM);
2627                 isp_rq_check_above(isp);
2628                 break;
2629 #endif
2630         case XPT_RESET_DEV:             /* BDR the specified SCSI device */
2631                 tgt = ccb->ccb_h.target_id;
2632                 tgt |= (bus << 16);
2633
2634                 error = isp_control(isp, ISPCTL_RESET_DEV, bus, tgt);
2635                 if (error) {
2636                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2637                 } else {
2638                         /*
2639                          * If we have a FC device, reset the Command
2640                          * Reference Number, because the target will expect
2641                          * that we re-start the CRN at 1 after a reset.
2642                          */
2643                         isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1);
2644
2645                         ccb->ccb_h.status = CAM_REQ_CMP;
2646                 }
2647                 xpt_done(ccb);
2648                 break;
2649         case XPT_ABORT:                 /* Abort the specified CCB */
2650         {
2651                 union ccb *accb = ccb->cab.abort_ccb;
2652                 switch (accb->ccb_h.func_code) {
2653 #ifdef  ISP_TARGET_MODE
2654                 case XPT_ACCEPT_TARGET_IO:
2655                         isp_abort_atio(isp, ccb);
2656                         break;
2657                 case XPT_IMMEDIATE_NOTIFY:
2658                         isp_abort_inot(isp, ccb);
2659                         break;
2660 #endif
2661                 case XPT_SCSI_IO:
2662                         error = isp_control(isp, ISPCTL_ABORT_CMD, accb);
2663                         if (error) {
2664                                 ccb->ccb_h.status = CAM_UA_ABORT;
2665                         } else {
2666                                 ccb->ccb_h.status = CAM_REQ_CMP;
2667                         }
2668                         break;
2669                 default:
2670                         ccb->ccb_h.status = CAM_REQ_INVALID;
2671                         break;
2672                 }
2673                 /*
2674                  * This is not a queued CCB, so the caller expects it to be
2675                  * complete when control is returned.
2676                  */
2677                 break;
2678         }
2679 #define IS_CURRENT_SETTINGS(c)  (c->type == CTS_TYPE_CURRENT_SETTINGS)
2680         case XPT_SET_TRAN_SETTINGS:     /* Nexus Settings */
2681                 cts = &ccb->cts;
2682                 if (!IS_CURRENT_SETTINGS(cts)) {
2683                         ccb->ccb_h.status = CAM_REQ_INVALID;
2684                         xpt_done(ccb);
2685                         break;
2686                 }
2687                 ccb->ccb_h.status = CAM_REQ_CMP;
2688                 xpt_done(ccb);
2689                 break;
2690         case XPT_GET_TRAN_SETTINGS:
2691         {
2692                 struct ccb_trans_settings_scsi *scsi;
2693                 struct ccb_trans_settings_fc *fc;
2694
2695                 cts = &ccb->cts;
2696                 scsi = &cts->proto_specific.scsi;
2697                 fc = &cts->xport_specific.fc;
2698                 tgt = cts->ccb_h.target_id;
2699                 fcp = FCPARAM(isp, bus);
2700
2701                 cts->protocol = PROTO_SCSI;
2702                 cts->protocol_version = SCSI_REV_2;
2703                 cts->transport = XPORT_FC;
2704                 cts->transport_version = 0;
2705
2706                 scsi->valid = CTS_SCSI_VALID_TQ;
2707                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2708                 fc->valid = CTS_FC_VALID_SPEED;
2709                 fc->bitrate = fcp->isp_gbspeed * 100000;
2710                 if (tgt < MAX_FC_TARG) {
2711                         fcportdb_t *lp = &fcp->portdb[tgt];
2712                         fc->wwnn = lp->node_wwn;
2713                         fc->wwpn = lp->port_wwn;
2714                         fc->port = lp->portid;
2715                         fc->valid |= CTS_FC_VALID_WWNN | CTS_FC_VALID_WWPN | CTS_FC_VALID_PORT;
2716                 }
2717                 ccb->ccb_h.status = CAM_REQ_CMP;
2718                 xpt_done(ccb);
2719                 break;
2720         }
2721         case XPT_CALC_GEOMETRY:
2722                 cam_calc_geometry(&ccb->ccg, 1);
2723                 xpt_done(ccb);
2724                 break;
2725
2726         case XPT_RESET_BUS:             /* Reset the specified bus */
2727                 error = isp_control(isp, ISPCTL_RESET_BUS, bus);
2728                 if (error) {
2729                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2730                         xpt_done(ccb);
2731                         break;
2732                 }
2733                 if (bootverbose) {
2734                         xpt_print(ccb->ccb_h.path, "reset bus on channel %d\n", bus);
2735                 }
2736                 xpt_async(AC_BUS_RESET, ISP_FC_PC(isp, bus)->path, 0);
2737                 ccb->ccb_h.status = CAM_REQ_CMP;
2738                 xpt_done(ccb);
2739                 break;
2740
2741         case XPT_TERM_IO:               /* Terminate the I/O process */
2742                 ccb->ccb_h.status = CAM_REQ_INVALID;
2743                 xpt_done(ccb);
2744                 break;
2745
2746         case XPT_SET_SIM_KNOB:          /* Set SIM knobs */
2747         {
2748                 struct ccb_sim_knob *kp = &ccb->knob;
2749                 fcparam *fcp = FCPARAM(isp, bus);
2750
2751                 if (kp->xport_specific.fc.valid & KNOB_VALID_ADDRESS) {
2752                         fcp->isp_wwnn = ISP_FC_PC(isp, bus)->def_wwnn = kp->xport_specific.fc.wwnn;
2753                         fcp->isp_wwpn = ISP_FC_PC(isp, bus)->def_wwpn = kp->xport_specific.fc.wwpn;
2754                         isp_prt(isp, ISP_LOGALL, "Setting Channel %d wwns to 0x%jx 0x%jx", bus, fcp->isp_wwnn, fcp->isp_wwpn);
2755                 }
2756                 ccb->ccb_h.status = CAM_REQ_CMP;
2757                 if (kp->xport_specific.fc.valid & KNOB_VALID_ROLE) {
2758                         int rchange = 0;
2759                         int newrole = 0;
2760
2761                         switch (kp->xport_specific.fc.role) {
2762                         case KNOB_ROLE_NONE:
2763                                 if (fcp->role != ISP_ROLE_NONE) {
2764                                         rchange = 1;
2765                                         newrole = ISP_ROLE_NONE;
2766                                 }
2767                                 break;
2768                         case KNOB_ROLE_TARGET:
2769                                 if (fcp->role != ISP_ROLE_TARGET) {
2770                                         rchange = 1;
2771                                         newrole = ISP_ROLE_TARGET;
2772                                 }
2773                                 break;
2774                         case KNOB_ROLE_INITIATOR:
2775                                 if (fcp->role != ISP_ROLE_INITIATOR) {
2776                                         rchange = 1;
2777                                         newrole = ISP_ROLE_INITIATOR;
2778                                 }
2779                                 break;
2780                         case KNOB_ROLE_BOTH:
2781                                 if (fcp->role != ISP_ROLE_BOTH) {
2782                                         rchange = 1;
2783                                         newrole = ISP_ROLE_BOTH;
2784                                 }
2785                                 break;
2786                         }
2787                         if (rchange) {
2788                                 ISP_PATH_PRT(isp, ISP_LOGCONFIG, ccb->ccb_h.path, "changing role on from %d to %d\n", fcp->role, newrole);
2789                                 if (isp_control(isp, ISPCTL_CHANGE_ROLE,
2790                                     bus, newrole) != 0) {
2791                                         ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2792                                         xpt_done(ccb);
2793                                         break;
2794                                 }
2795                         }
2796                 }
2797                 xpt_done(ccb);
2798                 break;
2799         }
2800         case XPT_GET_SIM_KNOB_OLD:      /* Get SIM knobs -- compat value */
2801         case XPT_GET_SIM_KNOB:          /* Get SIM knobs */
2802         {
2803                 struct ccb_sim_knob *kp = &ccb->knob;
2804                 fcparam *fcp = FCPARAM(isp, bus);
2805
2806                 kp->xport_specific.fc.wwnn = fcp->isp_wwnn;
2807                 kp->xport_specific.fc.wwpn = fcp->isp_wwpn;
2808                 switch (fcp->role) {
2809                 case ISP_ROLE_NONE:
2810                         kp->xport_specific.fc.role = KNOB_ROLE_NONE;
2811                         break;
2812                 case ISP_ROLE_TARGET:
2813                         kp->xport_specific.fc.role = KNOB_ROLE_TARGET;
2814                         break;
2815                 case ISP_ROLE_INITIATOR:
2816                         kp->xport_specific.fc.role = KNOB_ROLE_INITIATOR;
2817                         break;
2818                 case ISP_ROLE_BOTH:
2819                         kp->xport_specific.fc.role = KNOB_ROLE_BOTH;
2820                         break;
2821                 }
2822                 kp->xport_specific.fc.valid = KNOB_VALID_ADDRESS | KNOB_VALID_ROLE;
2823                 ccb->ccb_h.status = CAM_REQ_CMP;
2824                 xpt_done(ccb);
2825                 break;
2826         }
2827         case XPT_PATH_INQ:              /* Path routing inquiry */
2828         {
2829                 struct ccb_pathinq *cpi = &ccb->cpi;
2830
2831                 cpi->version_num = 1;
2832 #ifdef  ISP_TARGET_MODE
2833                 cpi->target_sprt = PIT_PROCESSOR | PIT_DISCONNECT | PIT_TERM_IO;
2834 #else
2835                 cpi->target_sprt = 0;
2836 #endif
2837                 cpi->hba_eng_cnt = 0;
2838                 cpi->max_target = ISP_MAX_TARGETS(isp) - 1;
2839                 cpi->max_lun = 255;
2840                 cpi->bus_id = cam_sim_bus(sim);
2841                 cpi->maxio = (ISP_NSEG64_MAX - 1) * PAGE_SIZE;
2842
2843                 fcp = FCPARAM(isp, bus);
2844
2845                 cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
2846                 cpi->hba_misc |= PIM_EXTLUNS | PIM_NOSCAN;
2847
2848                 /*
2849                  * Because our loop ID can shift from time to time,
2850                  * make our initiator ID out of range of our bus.
2851                  */
2852                 cpi->initiator_id = cpi->max_target + 1;
2853
2854                 /*
2855                  * Set base transfer capabilities for Fibre Channel, for this HBA.
2856                  */
2857                 if (IS_25XX(isp))
2858                         cpi->base_transfer_speed = 8000000;
2859                 else
2860                         cpi->base_transfer_speed = 4000000;
2861                 cpi->hba_inquiry = PI_TAG_ABLE;
2862                 cpi->transport = XPORT_FC;
2863                 cpi->transport_version = 0;
2864                 cpi->xport_specific.fc.wwnn = fcp->isp_wwnn;
2865                 cpi->xport_specific.fc.wwpn = fcp->isp_wwpn;
2866                 cpi->xport_specific.fc.port = fcp->isp_portid;
2867                 cpi->xport_specific.fc.bitrate = fcp->isp_gbspeed * 1000;
2868                 cpi->protocol = PROTO_SCSI;
2869                 cpi->protocol_version = SCSI_REV_2;
2870                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2871                 strlcpy(cpi->hba_vid, "Qlogic", HBA_IDLEN);
2872                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2873                 cpi->unit_number = cam_sim_unit(sim);
2874                 cpi->ccb_h.status = CAM_REQ_CMP;
2875                 xpt_done(ccb);
2876                 break;
2877         }
2878         default:
2879                 ccb->ccb_h.status = CAM_REQ_INVALID;
2880                 xpt_done(ccb);
2881                 break;
2882         }
2883 }
2884
2885 void
2886 isp_done(XS_T *sccb)
2887 {
2888         ispsoftc_t *isp = XS_ISP(sccb);
2889         uint32_t status;
2890
2891         if (XS_NOERR(sccb))
2892                 XS_SETERR(sccb, CAM_REQ_CMP);
2893
2894         if ((sccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP && (sccb->scsi_status != SCSI_STATUS_OK)) {
2895                 sccb->ccb_h.status &= ~CAM_STATUS_MASK;
2896                 if ((sccb->scsi_status == SCSI_STATUS_CHECK_COND) && (sccb->ccb_h.status & CAM_AUTOSNS_VALID) == 0) {
2897                         sccb->ccb_h.status |= CAM_AUTOSENSE_FAIL;
2898                 } else {
2899                         sccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
2900                 }
2901         }
2902
2903         sccb->ccb_h.status &= ~CAM_SIM_QUEUED;
2904         status = sccb->ccb_h.status & CAM_STATUS_MASK;
2905         if (status != CAM_REQ_CMP &&
2906             (sccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2907                 sccb->ccb_h.status |= CAM_DEV_QFRZN;
2908                 xpt_freeze_devq(sccb->ccb_h.path, 1);
2909         }
2910
2911         if (ISP_PCMD(sccb)) {
2912                 if (callout_active(&PISP_PCMD(sccb)->wdog))
2913                         callout_stop(&PISP_PCMD(sccb)->wdog);
2914                 isp_free_pcmd(isp, (union ccb *) sccb);
2915         }
2916         isp_rq_check_below(isp);
2917         xpt_done((union ccb *) sccb);
2918 }
2919
2920 void
2921 isp_async(ispsoftc_t *isp, ispasync_t cmd, ...)
2922 {
2923         int bus;
2924         static const char prom[] = "Chan %d [%d] WWPN 0x%16jx PortID 0x%06x handle 0x%x %s %s";
2925         char buf[64];
2926         char *msg = NULL;
2927         target_id_t tgt = 0;
2928         fcportdb_t *lp;
2929         struct isp_fc *fc;
2930         struct ac_contract ac;
2931         struct ac_device_changed *adc;
2932         va_list ap;
2933
2934         switch (cmd) {
2935         case ISPASYNC_BUS_RESET:
2936         {
2937                 va_start(ap, cmd);
2938                 bus = va_arg(ap, int);
2939                 va_end(ap);
2940                 isp_prt(isp, ISP_LOGINFO, "SCSI bus reset on bus %d detected", bus);
2941                 xpt_async(AC_BUS_RESET, ISP_FC_PC(isp, bus)->path, NULL);
2942                 break;
2943         }
2944         case ISPASYNC_LOOP_RESET:
2945         {
2946                 uint16_t lipp;
2947                 fcparam *fcp;
2948                 va_start(ap, cmd);
2949                 bus = va_arg(ap, int);
2950                 va_end(ap);
2951
2952                 lipp = ISP_READ(isp, OUTMAILBOX1);
2953                 fcp = FCPARAM(isp, bus);
2954
2955                 isp_prt(isp, ISP_LOGINFO, "Chan %d LOOP Reset, LIP primitive %x", bus, lipp);
2956                 /*
2957                  * Per FCP-4, a Reset LIP should result in a CRN reset. Other
2958                  * LIPs and loop up/down events should never reset the CRN. For
2959                  * an as of yet unknown reason, 24xx series cards (and
2960                  * potentially others) can interrupt with a LIP Reset status
2961                  * when no LIP reset came down the wire. Additionally, the LIP
2962                  * primitive accompanying this status would not be a valid LIP
2963                  * Reset primitive, but some variation of an invalid AL_PA
2964                  * LIP. As a result, we have to verify the AL_PD in the LIP
2965                  * addresses our port before blindly resetting.
2966                 */
2967                 if (FCP_IS_DEST_ALPD(fcp, (lipp & 0x00FF)))
2968                         isp_fcp_reset_crn(isp, bus, /*tgt*/0, /*tgt_set*/ 0);
2969                 isp_loop_changed(isp, bus);
2970                 break;
2971         }
2972         case ISPASYNC_LIP:
2973                 if (msg == NULL)
2974                         msg = "LIP Received";
2975                 /* FALLTHROUGH */
2976         case ISPASYNC_LOOP_DOWN:
2977                 if (msg == NULL)
2978                         msg = "LOOP Down";
2979                 /* FALLTHROUGH */
2980         case ISPASYNC_LOOP_UP:
2981                 if (msg == NULL)
2982                         msg = "LOOP Up";
2983                 va_start(ap, cmd);
2984                 bus = va_arg(ap, int);
2985                 va_end(ap);
2986                 isp_loop_changed(isp, bus);
2987                 isp_prt(isp, ISP_LOGINFO, "Chan %d %s", bus, msg);
2988                 break;
2989         case ISPASYNC_DEV_ARRIVED:
2990                 va_start(ap, cmd);
2991                 bus = va_arg(ap, int);
2992                 lp = va_arg(ap, fcportdb_t *);
2993                 va_end(ap);
2994                 fc = ISP_FC_PC(isp, bus);
2995                 tgt = FC_PORTDB_TGT(isp, bus, lp);
2996                 isp_gen_role_str(buf, sizeof (buf), lp->prli_word3);
2997                 isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "arrived");
2998                 if ((FCPARAM(isp, bus)->role & ISP_ROLE_INITIATOR) &&
2999                     (lp->prli_word3 & PRLI_WD3_TARGET_FUNCTION)) {
3000                         lp->is_target = 1;
3001                         isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1);
3002                         isp_make_here(isp, lp, bus, tgt);
3003                 }
3004                 if ((FCPARAM(isp, bus)->role & ISP_ROLE_TARGET) &&
3005                     (lp->prli_word3 & PRLI_WD3_INITIATOR_FUNCTION)) {
3006                         lp->is_initiator = 1;
3007                         ac.contract_number = AC_CONTRACT_DEV_CHG;
3008                         adc = (struct ac_device_changed *) ac.contract_data;
3009                         adc->wwpn = lp->port_wwn;
3010                         adc->port = lp->portid;
3011                         adc->target = tgt;
3012                         adc->arrived = 1;
3013                         xpt_async(AC_CONTRACT, fc->path, &ac);
3014                 }
3015                 break;
3016         case ISPASYNC_DEV_CHANGED:
3017         case ISPASYNC_DEV_STAYED:
3018         {
3019                 int crn_reset_done;
3020
3021                 crn_reset_done = 0;
3022                 va_start(ap, cmd);
3023                 bus = va_arg(ap, int);
3024                 lp = va_arg(ap, fcportdb_t *);
3025                 va_end(ap);
3026                 fc = ISP_FC_PC(isp, bus);
3027                 tgt = FC_PORTDB_TGT(isp, bus, lp);
3028                 isp_gen_role_str(buf, sizeof (buf), lp->new_prli_word3);
3029                 if (cmd == ISPASYNC_DEV_CHANGED)
3030                         isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->new_portid, lp->handle, buf, "changed");
3031                 else
3032                         isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "stayed");
3033
3034                 if (lp->is_target !=
3035                     ((FCPARAM(isp, bus)->role & ISP_ROLE_INITIATOR) &&
3036                      (lp->new_prli_word3 & PRLI_WD3_TARGET_FUNCTION))) {
3037                         lp->is_target = !lp->is_target;
3038                         if (lp->is_target) {
3039                                 if (cmd == ISPASYNC_DEV_CHANGED) {
3040                                         isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1);
3041                                         crn_reset_done = 1;
3042                                 }
3043                                 isp_make_here(isp, lp, bus, tgt);
3044                         } else {
3045                                 isp_make_gone(isp, lp, bus, tgt);
3046                                 if (cmd == ISPASYNC_DEV_CHANGED) {
3047                                         isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1);
3048                                         crn_reset_done = 1;
3049                                 }
3050                         }
3051                 }
3052                 if (lp->is_initiator !=
3053                     ((FCPARAM(isp, bus)->role & ISP_ROLE_TARGET) &&
3054                      (lp->new_prli_word3 & PRLI_WD3_INITIATOR_FUNCTION))) {
3055                         lp->is_initiator = !lp->is_initiator;
3056                         ac.contract_number = AC_CONTRACT_DEV_CHG;
3057                         adc = (struct ac_device_changed *) ac.contract_data;
3058                         adc->wwpn = lp->port_wwn;
3059                         adc->port = lp->portid;
3060                         adc->target = tgt;
3061                         adc->arrived = lp->is_initiator;
3062                         xpt_async(AC_CONTRACT, fc->path, &ac);
3063                 }
3064
3065                 if ((cmd == ISPASYNC_DEV_CHANGED) &&
3066                     (crn_reset_done == 0))
3067                         isp_fcp_reset_crn(isp, bus, tgt, /*tgt_set*/ 1);
3068
3069                 break;
3070         }
3071         case ISPASYNC_DEV_GONE:
3072                 va_start(ap, cmd);
3073                 bus = va_arg(ap, int);
3074                 lp = va_arg(ap, fcportdb_t *);
3075                 va_end(ap);
3076                 fc = ISP_FC_PC(isp, bus);
3077                 tgt = FC_PORTDB_TGT(isp, bus, lp);
3078                 /*
3079                  * If this has a virtual target or initiator set the isp_gdt
3080                  * timer running on it to delay its departure.
3081                  */
3082                 isp_gen_role_str(buf, sizeof (buf), lp->prli_word3);
3083                 if (lp->is_target || lp->is_initiator) {
3084                         lp->state = FC_PORTDB_STATE_ZOMBIE;
3085                         lp->gone_timer = fc->gone_device_time;
3086                         isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "gone zombie");
3087                         if (fc->ready && !callout_active(&fc->gdt)) {
3088                                 isp_prt(isp, ISP_LOG_SANCFG|ISP_LOGDEBUG0, "Chan %d Starting Gone Device Timer with %u seconds time now %lu", bus, lp->gone_timer, (unsigned long)time_uptime);
3089                                 callout_reset(&fc->gdt, hz, isp_gdt, fc);
3090                         }
3091                         break;
3092                 }
3093                 isp_prt(isp, ISP_LOGCONFIG, prom, bus, tgt, lp->port_wwn, lp->portid, lp->handle, buf, "gone");
3094                 break;
3095         case ISPASYNC_CHANGE_NOTIFY:
3096         {
3097                 char *msg;
3098                 int evt, nphdl, nlstate, portid, reason;
3099
3100                 va_start(ap, cmd);
3101                 bus = va_arg(ap, int);
3102                 evt = va_arg(ap, int);
3103                 if (evt == ISPASYNC_CHANGE_PDB) {
3104                         nphdl = va_arg(ap, int);
3105                         nlstate = va_arg(ap, int);
3106                         reason = va_arg(ap, int);
3107                 } else if (evt == ISPASYNC_CHANGE_SNS) {
3108                         portid = va_arg(ap, int);
3109                 } else {
3110                         nphdl = NIL_HANDLE;
3111                         nlstate = reason = 0;
3112                 }
3113                 va_end(ap);
3114
3115                 if (evt == ISPASYNC_CHANGE_PDB) {
3116                         int tgt_set = 0;
3117                         msg = "Port Database Changed";
3118                         isp_prt(isp, ISP_LOGINFO,
3119                             "Chan %d %s (nphdl 0x%x state 0x%x reason 0x%x)",
3120                             bus, msg, nphdl, nlstate, reason);
3121                         /*
3122                          * Port database syncs are not sufficient for
3123                          * determining that logins or logouts are done on the
3124                          * loop, but this information is directly available from
3125                          * the reason code from the incoming mbox. We must reset
3126                          * the fcp crn on these events according to FCP-4
3127                          */
3128                         switch (reason) {
3129                         case PDB24XX_AE_IMPL_LOGO_1:
3130                         case PDB24XX_AE_IMPL_LOGO_2:
3131                         case PDB24XX_AE_IMPL_LOGO_3:
3132                         case PDB24XX_AE_PLOGI_RCVD:
3133                         case PDB24XX_AE_PRLI_RCVD:
3134                         case PDB24XX_AE_PRLO_RCVD:
3135                         case PDB24XX_AE_LOGO_RCVD:
3136                         case PDB24XX_AE_PLOGI_DONE:
3137                         case PDB24XX_AE_PRLI_DONE:
3138                                 /*
3139                                  * If the event is not global, twiddle tgt and
3140                                  * tgt_set to nominate only the target
3141                                  * associated with the nphdl.
3142                                  */
3143                                 if (nphdl != PDB24XX_AE_GLOBAL) {
3144                                         /* Break if we don't yet have the pdb */
3145                                         if (!isp_find_pdb_by_handle(isp, bus, nphdl, &lp))
3146                                                 break;
3147                                         tgt = FC_PORTDB_TGT(isp, bus, lp);
3148                                         tgt_set = 1;
3149                                 }
3150                                 isp_fcp_reset_crn(isp, bus, tgt, tgt_set);
3151                                 break;
3152                         default:
3153                                 break; /* NOP */
3154                         }
3155                 } else if (evt == ISPASYNC_CHANGE_SNS) {
3156                         msg = "Name Server Database Changed";
3157                         isp_prt(isp, ISP_LOGINFO, "Chan %d %s (PortID 0x%06x)",
3158                             bus, msg, portid);
3159                 } else {
3160                         msg = "Other Change Notify";
3161                         isp_prt(isp, ISP_LOGINFO, "Chan %d %s", bus, msg);
3162                 }
3163                 isp_loop_changed(isp, bus);
3164                 break;
3165         }
3166 #ifdef  ISP_TARGET_MODE
3167         case ISPASYNC_TARGET_NOTIFY:
3168         {
3169                 isp_notify_t *notify;
3170                 va_start(ap, cmd);
3171                 notify = va_arg(ap, isp_notify_t *);
3172                 va_end(ap);
3173                 switch (notify->nt_ncode) {
3174                 case NT_ABORT_TASK:
3175                 case NT_ABORT_TASK_SET:
3176                 case NT_CLEAR_ACA:
3177                 case NT_CLEAR_TASK_SET:
3178                 case NT_LUN_RESET:
3179                 case NT_TARGET_RESET:
3180                 case NT_QUERY_TASK_SET:
3181                 case NT_QUERY_ASYNC_EVENT:
3182                         /*
3183                          * These are task management functions.
3184                          */
3185                         isp_handle_platform_target_tmf(isp, notify);
3186                         break;
3187                 case NT_BUS_RESET:
3188                 case NT_LIP_RESET:
3189                 case NT_LINK_UP:
3190                 case NT_LINK_DOWN:
3191                 case NT_HBA_RESET:
3192                         /*
3193                          * No action need be taken here.
3194                          */
3195                         break;
3196                 case NT_GLOBAL_LOGOUT:
3197                 case NT_LOGOUT:
3198                         /*
3199                          * This is device arrival/departure notification
3200                          */
3201                         isp_handle_platform_target_notify_ack(isp, notify, 0);
3202                         break;
3203                 case NT_SRR:
3204                         isp_handle_platform_srr(isp, notify);
3205                         break;
3206                 default:
3207                         isp_prt(isp, ISP_LOGALL, "target notify code 0x%x", notify->nt_ncode);
3208                         isp_handle_platform_target_notify_ack(isp, notify, 0);
3209                         break;
3210                 }
3211                 break;
3212         }
3213         case ISPASYNC_TARGET_NOTIFY_ACK:
3214         {
3215                 void *inot;
3216                 va_start(ap, cmd);
3217                 inot = va_arg(ap, void *);
3218                 va_end(ap);
3219                 if (isp_notify_ack(isp, inot)) {
3220                         isp_tna_t *tp = malloc(sizeof (*tp), M_DEVBUF, M_NOWAIT);
3221                         if (tp) {
3222                                 tp->isp = isp;
3223                                 memcpy(tp->data, inot, sizeof (tp->data));
3224                                 tp->not = tp->data;
3225                                 callout_init_mtx(&tp->timer, &isp->isp_lock, 0);
3226                                 callout_reset(&tp->timer, 5,
3227                                     isp_refire_notify_ack, tp);
3228                         } else {
3229                                 isp_prt(isp, ISP_LOGERR, "you lose- cannot allocate a notify refire");
3230                         }
3231                 }
3232                 break;
3233         }
3234         case ISPASYNC_TARGET_ACTION:
3235         {
3236                 isphdr_t *hp;
3237
3238                 va_start(ap, cmd);
3239                 hp = va_arg(ap, isphdr_t *);
3240                 va_end(ap);
3241                 switch (hp->rqs_entry_type) {
3242                 case RQSTYPE_ATIO:
3243                         isp_handle_platform_atio7(isp, (at7_entry_t *)hp);
3244                         break;
3245                 case RQSTYPE_CTIO7:
3246                         isp_handle_platform_ctio(isp, (ct7_entry_t *)hp);
3247                         break;
3248                 default:
3249                         isp_prt(isp, ISP_LOGWARN, "%s: unhandled target action 0x%x",
3250                             __func__, hp->rqs_entry_type);
3251                         break;
3252                 }
3253                 break;
3254         }
3255 #endif
3256         case ISPASYNC_FW_CRASH:
3257         {
3258                 uint16_t mbox1;
3259                 mbox1 = ISP_READ(isp, OUTMAILBOX1);
3260                 isp_prt(isp, ISP_LOGERR, "Internal Firmware Error @ RISC Address 0x%x", mbox1);
3261 #if 0
3262                 mbox1 = isp->isp_osinfo.mbox_sleep_ok;
3263                 isp->isp_osinfo.mbox_sleep_ok = 0;
3264                 isp_reinit(isp, 1);
3265                 isp->isp_osinfo.mbox_sleep_ok = mbox1;
3266                 isp_async(isp, ISPASYNC_FW_RESTARTED, NULL);
3267 #endif
3268                 break;
3269         }
3270         default:
3271                 isp_prt(isp, ISP_LOGERR, "unknown isp_async event %d", cmd);
3272                 break;
3273         }
3274 }
3275
3276 uint64_t
3277 isp_default_wwn(ispsoftc_t * isp, int chan, int isactive, int iswwnn)
3278 {
3279         uint64_t seed;
3280         struct isp_fc *fc = ISP_FC_PC(isp, chan);
3281
3282         /* First try to use explicitly configured WWNs. */
3283         seed = iswwnn ? fc->def_wwnn : fc->def_wwpn;
3284         if (seed)
3285                 return (seed);
3286
3287         /* Otherwise try to use WWNs from NVRAM. */
3288         if (isactive) {
3289                 seed = iswwnn ? FCPARAM(isp, chan)->isp_wwnn_nvram :
3290                     FCPARAM(isp, chan)->isp_wwpn_nvram;
3291                 if (seed)
3292                         return (seed);
3293         }
3294
3295         /* If still no WWNs, try to steal them from the first channel. */
3296         if (chan > 0) {
3297                 seed = iswwnn ? ISP_FC_PC(isp, 0)->def_wwnn :
3298                     ISP_FC_PC(isp, 0)->def_wwpn;
3299                 if (seed == 0) {
3300                         seed = iswwnn ? FCPARAM(isp, 0)->isp_wwnn_nvram :
3301                             FCPARAM(isp, 0)->isp_wwpn_nvram;
3302                 }
3303         }
3304
3305         /* If still nothing -- improvise. */
3306         if (seed == 0) {
3307                 seed = 0x400000007F000000ull + device_get_unit(isp->isp_dev);
3308                 if (!iswwnn)
3309                         seed ^= 0x0100000000000000ULL;
3310         }
3311
3312         /* For additional channels we have to improvise even more. */
3313         if (!iswwnn && chan > 0) {
3314                 /*
3315                  * We'll stick our channel number plus one first into bits
3316                  * 57..59 and thence into bits 52..55 which allows for 8 bits
3317                  * of channel which is enough for our maximum of 255 channels.
3318                  */
3319                 seed ^= 0x0100000000000000ULL;
3320                 seed ^= ((uint64_t) (chan + 1) & 0xf) << 56;
3321                 seed ^= ((uint64_t) ((chan + 1) >> 4) & 0xf) << 52;
3322         }
3323         return (seed);
3324 }
3325
3326 void
3327 isp_prt(ispsoftc_t *isp, int level, const char *fmt, ...)
3328 {
3329         int loc;
3330         char lbuf[200];
3331         va_list ap;
3332
3333         if (level != ISP_LOGALL && (level & isp->isp_dblev) == 0) {
3334                 return;
3335         }
3336         snprintf(lbuf, sizeof (lbuf), "%s: ", device_get_nameunit(isp->isp_dev));
3337         loc = strlen(lbuf);
3338         va_start(ap, fmt);
3339         vsnprintf(&lbuf[loc], sizeof (lbuf) - loc - 1, fmt, ap);
3340         va_end(ap);
3341         printf("%s\n", lbuf);
3342 }
3343
3344 void
3345 isp_xs_prt(ispsoftc_t *isp, XS_T *xs, int level, const char *fmt, ...)
3346 {
3347         va_list ap;
3348         if (level != ISP_LOGALL && (level & isp->isp_dblev) == 0) {
3349                 return;
3350         }
3351         xpt_print_path(xs->ccb_h.path);
3352         va_start(ap, fmt);
3353         vprintf(fmt, ap);
3354         va_end(ap);
3355         printf("\n");
3356 }
3357
3358 uint64_t
3359 isp_nanotime_sub(struct timespec *b, struct timespec *a)
3360 {
3361         uint64_t elapsed;
3362         struct timespec x;
3363
3364         timespecsub(b, a, &x);
3365         elapsed = GET_NANOSEC(&x);
3366         if (elapsed == 0)
3367                 elapsed++;
3368         return (elapsed);
3369 }
3370
3371 int
3372 isp_mbox_acquire(ispsoftc_t *isp)
3373 {
3374         if (isp->isp_osinfo.mboxbsy) {
3375                 return (1);
3376         } else {
3377                 isp->isp_osinfo.mboxcmd_done = 0;
3378                 isp->isp_osinfo.mboxbsy = 1;
3379                 return (0);
3380         }
3381 }
3382
3383 void
3384 isp_mbox_wait_complete(ispsoftc_t *isp, mbreg_t *mbp)
3385 {
3386         u_int t, to;
3387
3388         to = (mbp->timeout == 0) ? MBCMD_DEFAULT_TIMEOUT : mbp->timeout;
3389         if (isp->isp_osinfo.mbox_sleep_ok) {
3390                 isp->isp_osinfo.mbox_sleep_ok = 0;
3391                 isp->isp_osinfo.mbox_sleeping = 1;
3392                 msleep_sbt(&isp->isp_osinfo.mboxcmd_done, &isp->isp_lock,
3393                     PRIBIO, "ispmbx_sleep", to * SBT_1US, 0, 0);
3394                 isp->isp_osinfo.mbox_sleep_ok = 1;
3395                 isp->isp_osinfo.mbox_sleeping = 0;
3396         } else {
3397                 for (t = 0; t < to; t += 100) {
3398                         if (isp->isp_osinfo.mboxcmd_done)
3399                                 break;
3400                         ISP_RUN_ISR(isp);
3401                         if (isp->isp_osinfo.mboxcmd_done)
3402                                 break;
3403                         ISP_DELAY(100);
3404                 }
3405         }
3406         if (isp->isp_osinfo.mboxcmd_done == 0) {
3407                 isp_prt(isp, ISP_LOGWARN, "%s Mailbox Command (0x%x) Timeout (%uus) (%s:%d)",
3408                     isp->isp_osinfo.mbox_sleep_ok? "Interrupting" : "Polled",
3409                     isp->isp_lastmbxcmd, to, mbp->func, mbp->lineno);
3410                 mbp->param[0] = MBOX_TIMEOUT;
3411                 isp->isp_osinfo.mboxcmd_done = 1;
3412         }
3413 }
3414
3415 void
3416 isp_mbox_notify_done(ispsoftc_t *isp)
3417 {
3418         isp->isp_osinfo.mboxcmd_done = 1;
3419         if (isp->isp_osinfo.mbox_sleeping)
3420                 wakeup(&isp->isp_osinfo.mboxcmd_done);
3421 }
3422
3423 void
3424 isp_mbox_release(ispsoftc_t *isp)
3425 {
3426         isp->isp_osinfo.mboxbsy = 0;
3427 }
3428
3429 int
3430 isp_fc_scratch_acquire(ispsoftc_t *isp, int chan)
3431 {
3432         int ret = 0;
3433         if (isp->isp_osinfo.pc.fc[chan].fcbsy) {
3434                 ret = -1;
3435         } else {
3436                 isp->isp_osinfo.pc.fc[chan].fcbsy = 1;
3437         }
3438         return (ret);
3439 }
3440
3441 void
3442 isp_platform_intr(void *arg)
3443 {
3444         ispsoftc_t *isp = arg;
3445
3446         ISP_LOCK(isp);
3447         ISP_RUN_ISR(isp);
3448         ISP_UNLOCK(isp);
3449 }
3450
3451 void
3452 isp_platform_intr_resp(void *arg)
3453 {
3454         ispsoftc_t *isp = arg;
3455
3456         ISP_LOCK(isp);
3457         isp_intr_respq(isp);
3458         ISP_UNLOCK(isp);
3459
3460         /* We have handshake enabled, so explicitly complete interrupt */
3461         ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RISC_INT);
3462 }
3463
3464 void
3465 isp_platform_intr_atio(void *arg)
3466 {
3467         ispsoftc_t *isp = arg;
3468
3469         ISP_LOCK(isp);
3470 #ifdef  ISP_TARGET_MODE
3471         isp_intr_atioq(isp);
3472 #endif
3473         ISP_UNLOCK(isp);
3474
3475         /* We have handshake enabled, so explicitly complete interrupt */
3476         ISP_WRITE(isp, BIU2400_HCCR, HCCR_2400_CMD_CLEAR_RISC_INT);
3477 }
3478
3479 typedef struct {
3480         ispsoftc_t              *isp;
3481         struct ccb_scsiio       *csio;
3482         void                    *qe;
3483         int                     error;
3484 } mush_t;
3485
3486 static void
3487 isp_dma2(void *arg, bus_dma_segment_t *dm_segs, int nseg, int error)
3488 {
3489         mush_t *mp = (mush_t *) arg;
3490         ispsoftc_t *isp= mp->isp;
3491         struct ccb_scsiio *csio = mp->csio;
3492         bus_dmasync_op_t op;
3493
3494         if (error) {
3495                 mp->error = error;
3496                 return;
3497         }
3498         if ((csio->ccb_h.func_code == XPT_CONT_TARGET_IO) ^
3499             ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN))
3500                 op = BUS_DMASYNC_PREREAD;
3501         else
3502                 op = BUS_DMASYNC_PREWRITE;
3503         bus_dmamap_sync(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap, op);
3504
3505         mp->error = ISP_SEND_CMD(isp, mp->qe, dm_segs, nseg);
3506         if (mp->error)
3507                 isp_dmafree(isp, csio);
3508 }
3509
3510 int
3511 isp_dmasetup(ispsoftc_t *isp, struct ccb_scsiio *csio, void *qe)
3512 {
3513         mush_t mp;
3514         int error;
3515
3516         if (XS_XFRLEN(csio)) {
3517                 mp.isp = isp;
3518                 mp.csio = csio;
3519                 mp.qe = qe;
3520                 mp.error = 0;
3521                 error = bus_dmamap_load_ccb(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap,
3522                     (union ccb *)csio, isp_dma2, &mp, BUS_DMA_NOWAIT);
3523                 if (error == 0)
3524                         error = mp.error;
3525         } else {
3526                 error = ISP_SEND_CMD(isp, qe, NULL, 0);
3527         }
3528         switch (error) {
3529         case 0:
3530         case CMD_COMPLETE:
3531         case CMD_EAGAIN:
3532         case CMD_RQLATER:
3533                 break;
3534         case ENOMEM:
3535                 error = CMD_EAGAIN;
3536                 break;
3537         case EINVAL:
3538         case EFBIG:
3539                 csio->ccb_h.status = CAM_REQ_INVALID;
3540                 error = CMD_COMPLETE;
3541                 break;
3542         default:
3543                 csio->ccb_h.status = CAM_UNREC_HBA_ERROR;
3544                 error = CMD_COMPLETE;
3545                 break;
3546         }
3547         return (error);
3548 }
3549
3550 void
3551 isp_dmafree(ispsoftc_t *isp, struct ccb_scsiio *csio)
3552 {
3553         bus_dmasync_op_t op;
3554
3555         if (XS_XFRLEN(csio) == 0)
3556                 return;
3557
3558         if ((csio->ccb_h.func_code == XPT_CONT_TARGET_IO) ^
3559             ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN))
3560                 op = BUS_DMASYNC_POSTREAD;
3561         else
3562                 op = BUS_DMASYNC_POSTWRITE;
3563         bus_dmamap_sync(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap, op);
3564         bus_dmamap_unload(isp->isp_osinfo.dmat, PISP_PCMD(csio)->dmap);
3565 }
3566
3567 /*
3568  * Reset the command reference number for all LUNs on a specific target
3569  * (needed when a target arrives again) or for all targets on a port
3570  * (needed for events like a LIP).
3571  */
3572 void
3573 isp_fcp_reset_crn(ispsoftc_t *isp, int chan, uint32_t tgt, int tgt_set)
3574 {
3575         struct isp_fc *fc = ISP_FC_PC(isp, chan);
3576         struct isp_nexus *nxp;
3577         int i;
3578
3579         if (tgt_set == 0)
3580                 isp_prt(isp, ISP_LOGDEBUG0,
3581                     "Chan %d resetting CRN on all targets", chan);
3582         else
3583                 isp_prt(isp, ISP_LOGDEBUG0,
3584                     "Chan %d resetting CRN on target %u", chan, tgt);
3585
3586         for (i = 0; i < NEXUS_HASH_WIDTH; i++) {
3587                 for (nxp = fc->nexus_hash[i]; nxp != NULL; nxp = nxp->next) {
3588                         if (tgt_set == 0 || tgt == nxp->tgt)
3589                                 nxp->crnseed = 0;
3590                 }
3591         }
3592 }
3593
3594 int
3595 isp_fcp_next_crn(ispsoftc_t *isp, uint8_t *crnp, XS_T *cmd)
3596 {
3597         lun_id_t lun;
3598         uint32_t chan, tgt;
3599         struct isp_fc *fc;
3600         struct isp_nexus *nxp;
3601         int idx;
3602
3603         chan = XS_CHANNEL(cmd);
3604         tgt = XS_TGT(cmd);
3605         lun = XS_LUN(cmd);
3606         fc = &isp->isp_osinfo.pc.fc[chan];
3607         idx = NEXUS_HASH(tgt, lun);
3608         nxp = fc->nexus_hash[idx];
3609
3610         while (nxp) {
3611                 if (nxp->tgt == tgt && nxp->lun == lun)
3612                         break;
3613                 nxp = nxp->next;
3614         }
3615         if (nxp == NULL) {
3616                 nxp = fc->nexus_free_list;
3617                 if (nxp == NULL) {
3618                         nxp = malloc(sizeof (struct isp_nexus), M_DEVBUF, M_ZERO|M_NOWAIT);
3619                         if (nxp == NULL) {
3620                                 return (-1);
3621                         }
3622                 } else {
3623                         fc->nexus_free_list = nxp->next;
3624                 }
3625                 nxp->tgt = tgt;
3626                 nxp->lun = lun;
3627                 nxp->next = fc->nexus_hash[idx];
3628                 fc->nexus_hash[idx] = nxp;
3629         }
3630         if (nxp->crnseed == 0)
3631                 nxp->crnseed = 1;
3632         *crnp = nxp->crnseed++;
3633         return (0);
3634 }
3635
3636 /*
3637  * We enter with the lock held
3638  */
3639 void
3640 isp_timer(void *arg)
3641 {
3642         ispsoftc_t *isp = arg;
3643 #ifdef  ISP_TARGET_MODE
3644         isp_tmcmd_restart(isp);
3645 #endif
3646         callout_reset(&isp->isp_osinfo.tmo, isp_timer_count, isp_timer, isp);
3647 }
3648
3649 #ifdef  ISP_TARGET_MODE
3650 isp_ecmd_t *
3651 isp_get_ecmd(ispsoftc_t *isp)
3652 {
3653         isp_ecmd_t *ecmd = isp->isp_osinfo.ecmd_free;
3654         if (ecmd) {
3655                 isp->isp_osinfo.ecmd_free = ecmd->next;
3656         }
3657         return (ecmd);
3658 }
3659
3660 void
3661 isp_put_ecmd(ispsoftc_t *isp, isp_ecmd_t *ecmd)
3662 {
3663         ecmd->next = isp->isp_osinfo.ecmd_free;
3664         isp->isp_osinfo.ecmd_free = ecmd;
3665 }
3666 #endif