sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/condvar.h>
  36 #include <sys/conf.h>
  37 #include <sys/eventhandler.h>
  38 #include <sys/file.h>
  39 #include <sys/kernel.h>
  40 #include <sys/kthread.h>
  41 #include <sys/lock.h>
  42 #include <sys/malloc.h>
  43 #include <sys/mutex.h>
  44 #include <sys/module.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/systm.h>
  47 #include <sys/sx.h>
  48 #include <vm/uma.h>
  49
  50 #include <cam/cam.h>
  51 #include <cam/cam_ccb.h>
  52 #include <cam/cam_xpt.h>
  53 #include <cam/cam_debug.h>
  54 #include <cam/cam_sim.h>
  55 #include <cam/cam_xpt_sim.h>
  56 #include <cam/cam_xpt_periph.h>
  57 #include <cam/cam_periph.h>
  58 #include <cam/scsi/scsi_all.h>
  59 #include <cam/scsi/scsi_message.h>
  60
  61 #include <dev/iscsi/icl.h>
  62 #include <dev/iscsi/iscsi_ioctl.h>
  63 #include <dev/iscsi/iscsi_proto.h>
  64 #include <dev/iscsi/iscsi.h>
  65
  66 #ifdef ICL_KERNEL_PROXY
  67 #include <sys/socketvar.h>
  68 #endif
  69
  70 #ifdef ICL_KERNEL_PROXY
  71 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  72 #endif
  73
  74 /*
  75  * XXX: This is global so the iscsi_unload() can access it.
  76  *      Think about how to do this properly.
  77  */
  78 static struct iscsi_softc       *sc;
  79
  80 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  81 static int debug = 1;
  82 TUNABLE_INT("kern.iscsi.debug", &debug);
  83 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  84     &debug, 0, "Enable debug messages");
  85 static int ping_timeout = 5;
  86 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
  87 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  88     0, "Timeout for ping (NOP-Out) requests, in seconds");
  89 static int iscsid_timeout = 60;
  90 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
  91 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  92     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  93 static int login_timeout = 60;
  94 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
  95 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
  96     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  97 static int maxtags = 255;
  98 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
  99 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
 100     0, "Max number of IO requests queued");
 101 static int fail_on_disconnection = 0;
 102 TUNABLE_INT("kern.iscsi.fail_on_disconnection", &fail_on_disconnection);
 103 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
 104     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
 105
 106 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 107 static uma_zone_t iscsi_outstanding_zone;
 108
 109 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 110 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 111
 112 #define ISCSI_DEBUG(X, ...)                                             \
 113         do {                                                            \
 114                 if (debug > 1)                                          \
 115                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 116         } while (0)
 117
 118 #define ISCSI_WARN(X, ...)                                              \
 119         do {                                                            \
 120                 if (debug > 0) {                                        \
 121                         printf("WARNING: %s: " X "\n",                  \
 122                             __func__, ## __VA_ARGS__);                  \
 123                 }                                                       \
 124         } while (0)
 125
 126 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 127         do {                                                            \
 128                 if (debug > 1) {                                        \
 129                         printf("%s: %s (%s): " X "\n",                  \
 130                             __func__, S->is_conf.isc_target_addr,       \
 131                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 132                 }                                                       \
 133         } while (0)
 134
 135 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 136         do {                                                            \
 137                 if (debug > 0) {                                        \
 138                         printf("WARNING: %s (%s): " X "\n",             \
 139                             S->is_conf.isc_target_addr,                 \
 140                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 141                 }                                                       \
 142         } while (0)
 143
 144 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 145 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 146 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 147 #define ISCSI_SESSION_LOCK_ASSERT_NOT(X) mtx_assert(&X->is_lock, MA_NOTOWNED)
 148
 149 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 150                     int mode, struct thread *td);
 151
 152 static struct cdevsw iscsi_cdevsw = {
 153      .d_version = D_VERSION,
 154      .d_ioctl   = iscsi_ioctl,
 155      .d_name    = "iscsi",
 156 };
 157
 158 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 159 static void     iscsi_pdu_queue(struct icl_pdu *request);
 160 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 161 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 162 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 163 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
 164 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 165 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 166 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 167 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 168 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 169 static void     iscsi_session_reconnect(struct iscsi_session *is);
 170 static void     iscsi_session_terminate(struct iscsi_session *is);
 171 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 172 static void     iscsi_poll(struct cam_sim *sim);
 173 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 174                     uint32_t initiator_task_tag);
 175 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
 176                     uint32_t initiator_task_tag, union ccb *ccb);
 177 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 178                     struct iscsi_outstanding *io);
 179
 180 static bool
 181 iscsi_pdu_prepare(struct icl_pdu *request)
 182 {
 183         struct iscsi_session *is;
 184         struct iscsi_bhs_scsi_command *bhssc;
 185
 186         is = PDU_SESSION(request);
 187
 188         ISCSI_SESSION_LOCK_ASSERT(is);
 189
 190         /*
 191          * We're only using fields common for all the request
 192          * (initiator -> target) PDUs.
 193          */
 194         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 195
 196         /*
 197          * Data-Out PDU does not contain CmdSN.
 198          */
 199         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 200                 if (ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn) &&
 201                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 202                         /*
 203                          * Current MaxCmdSN prevents us from sending any more
 204                          * SCSI Command PDUs to the target; postpone the PDU.
 205                          * It will get resent by either iscsi_pdu_queue(),
 206                          * or by maintenance thread.
 207                          */
 208 #if 0
 209                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %u, "
 210                             "ExpCmdSN %u, MaxCmdSN %u, opcode 0x%x",
 211                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn,
 212                             bhssc->bhssc_opcode);
 213 #endif
 214                         return (true);
 215                 }
 216                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 217                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 218                         is->is_cmdsn++;
 219         }
 220         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 221
 222         return (false);
 223 }
 224
 225 static void
 226 iscsi_session_send_postponed(struct iscsi_session *is)
 227 {
 228         struct icl_pdu *request;
 229         bool postpone;
 230
 231         ISCSI_SESSION_LOCK_ASSERT(is);
 232
 233         while (!STAILQ_EMPTY(&is->is_postponed)) {
 234                 request = STAILQ_FIRST(&is->is_postponed);
 235                 postpone = iscsi_pdu_prepare(request);
 236                 if (postpone)
 237                         break;
 238                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 239                 icl_pdu_queue(request);
 240         }
 241 }
 242
 243 static void
 244 iscsi_pdu_queue_locked(struct icl_pdu *request)
 245 {
 246         struct iscsi_session *is;
 247         bool postpone;
 248
 249         is = PDU_SESSION(request);
 250         ISCSI_SESSION_LOCK_ASSERT(is);
 251         iscsi_session_send_postponed(is);
 252         postpone = iscsi_pdu_prepare(request);
 253         if (postpone) {
 254                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 255                 return;
 256         }
 257         icl_pdu_queue(request);
 258 }
 259
 260 static void
 261 iscsi_pdu_queue(struct icl_pdu *request)
 262 {
 263         struct iscsi_session *is;
 264
 265         is = PDU_SESSION(request);
 266         ISCSI_SESSION_LOCK(is);
 267         iscsi_pdu_queue_locked(request);
 268         ISCSI_SESSION_UNLOCK(is);
 269 }
 270
 271 static void
 272 iscsi_session_logout(struct iscsi_session *is)
 273 {
 274         struct icl_pdu *request;
 275         struct iscsi_bhs_logout_request *bhslr;
 276
 277         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 278         if (request == NULL)
 279                 return;
 280
 281         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 282         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 283         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 284         iscsi_pdu_queue_locked(request);
 285 }
 286
 287 static void
 288 iscsi_session_terminate_task(struct iscsi_session *is,
 289     struct iscsi_outstanding *io, bool requeue)
 290 {
 291
 292         if (io->io_ccb != NULL) {
 293                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
 294                 if (requeue)
 295                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 296                 else
 297                         io->io_ccb->ccb_h.status |= CAM_REQ_ABORTED;
 298                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 299                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 300                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 301                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 302                 }
 303                 xpt_done(io->io_ccb);
 304         }
 305         iscsi_outstanding_remove(is, io);
 306 }
 307
 308 static void
 309 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 310 {
 311         struct iscsi_outstanding *io, *tmp;
 312
 313         ISCSI_SESSION_LOCK_ASSERT(is);
 314
 315         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 316                 iscsi_session_terminate_task(is, io, requeue);
 317         }
 318 }
 319
 320 static void
 321 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 322 {
 323         struct icl_pdu *pdu;
 324
 325         ISCSI_SESSION_LOCK_ASSERT(is);
 326
 327         /*
 328          * Don't queue any new PDUs.
 329          */
 330         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 331                 ISCSI_SESSION_DEBUG(is, "freezing");
 332                 xpt_freeze_simq(is->is_sim, 1);
 333                 is->is_simq_frozen = true;
 334         }
 335
 336         /*
 337          * Remove postponed PDUs.
 338          */
 339         while (!STAILQ_EMPTY(&is->is_postponed)) {
 340                 pdu = STAILQ_FIRST(&is->is_postponed);
 341                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 342                 icl_pdu_free(pdu);
 343         }
 344
 345         if (destroy_sim == false) {
 346                 /*
 347                  * Terminate SCSI tasks, asking CAM to requeue them.
 348                  */
 349                 iscsi_session_terminate_tasks(is, true);
 350                 return;
 351         }
 352
 353         iscsi_session_terminate_tasks(is, false);
 354
 355         if (is->is_sim == NULL)
 356                 return;
 357
 358         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 359         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 360
 361         if (is->is_simq_frozen) {
 362                 xpt_release_simq(is->is_sim, 1);
 363                 is->is_simq_frozen = false;
 364         }
 365
 366         xpt_free_path(is->is_path);
 367         is->is_path = NULL;
 368         xpt_bus_deregister(cam_sim_path(is->is_sim));
 369         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 370         is->is_sim = NULL;
 371         is->is_devq = NULL;
 372 }
 373
 374 static void
 375 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 376 {
 377
 378         icl_conn_close(is->is_conn);
 379
 380         ISCSI_SESSION_LOCK(is);
 381
 382         is->is_connected = false;
 383         is->is_reconnecting = false;
 384         is->is_login_phase = false;
 385
 386 #ifdef ICL_KERNEL_PROXY
 387         if (is->is_login_pdu != NULL) {
 388                 icl_pdu_free(is->is_login_pdu);
 389                 is->is_login_pdu = NULL;
 390         }
 391         cv_signal(&is->is_login_cv);
 392 #endif
 393
 394         if (fail_on_disconnection) {
 395                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
 396                 iscsi_session_cleanup(is, true);
 397         } else {
 398                 iscsi_session_cleanup(is, false);
 399         }
 400
 401         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 402             ("destroying session with active tasks"));
 403         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 404             ("destroying session with postponed PDUs"));
 405
 406         /*
 407          * Request immediate reconnection from iscsid(8).
 408          */
 409         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 410         is->is_waiting_for_iscsid = true;
 411         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 412         is->is_timeout = 0;
 413         ISCSI_SESSION_UNLOCK(is);
 414         cv_signal(&is->is_softc->sc_cv);
 415 }
 416
 417 static void
 418 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 419 {
 420         struct iscsi_softc *sc;
 421
 422         sc = is->is_softc;
 423         sx_xlock(&sc->sc_lock);
 424         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 425         sx_xunlock(&sc->sc_lock);
 426
 427         icl_conn_close(is->is_conn);
 428
 429         ISCSI_SESSION_LOCK(is);
 430
 431         KASSERT(is->is_terminating, ("is_terminating == false"));
 432
 433 #ifdef ICL_KERNEL_PROXY
 434         if (is->is_login_pdu != NULL) {
 435                 icl_pdu_free(is->is_login_pdu);
 436                 is->is_login_pdu = NULL;
 437         }
 438         cv_signal(&is->is_login_cv);
 439 #endif
 440
 441         callout_drain(&is->is_callout);
 442
 443         iscsi_session_cleanup(is, true);
 444
 445         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 446             ("destroying session with active tasks"));
 447         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 448             ("destroying session with postponed PDUs"));
 449
 450         ISCSI_SESSION_UNLOCK(is);
 451
 452         icl_conn_free(is->is_conn);
 453         mtx_destroy(&is->is_lock);
 454         cv_destroy(&is->is_maintenance_cv);
 455 #ifdef ICL_KERNEL_PROXY
 456         cv_destroy(&is->is_login_cv);
 457 #endif
 458         ISCSI_SESSION_DEBUG(is, "terminated");
 459         free(is, M_ISCSI);
 460
 461         /*
 462          * The iscsi_unload() routine might be waiting.
 463          */
 464         cv_signal(&sc->sc_cv);
 465 }
 466
 467 static void
 468 iscsi_maintenance_thread(void *arg)
 469 {
 470         struct iscsi_session *is;
 471
 472         is = arg;
 473
 474         for (;;) {
 475                 ISCSI_SESSION_LOCK(is);
 476                 if (is->is_reconnecting == false &&
 477                     is->is_terminating == false &&
 478                     STAILQ_EMPTY(&is->is_postponed))
 479                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 480
 481                 if (is->is_reconnecting) {
 482                         ISCSI_SESSION_UNLOCK(is);
 483                         iscsi_maintenance_thread_reconnect(is);
 484                         continue;
 485                 }
 486
 487                 if (is->is_terminating) {
 488                         ISCSI_SESSION_UNLOCK(is);
 489                         iscsi_maintenance_thread_terminate(is);
 490                         kthread_exit();
 491                         return;
 492                 }
 493
 494                 iscsi_session_send_postponed(is);
 495                 ISCSI_SESSION_UNLOCK(is);
 496         }
 497 }
 498
 499 static void
 500 iscsi_session_reconnect(struct iscsi_session *is)
 501 {
 502
 503         /*
 504          * XXX: We can't use locking here, because
 505          *      it's being called from various contexts.
 506          *      Hope it doesn't break anything.
 507          */
 508         if (is->is_reconnecting)
 509                 return;
 510
 511         is->is_reconnecting = true;
 512         cv_signal(&is->is_maintenance_cv);
 513 }
 514
 515 static void
 516 iscsi_session_terminate(struct iscsi_session *is)
 517 {
 518         if (is->is_terminating)
 519                 return;
 520
 521         is->is_terminating = true;
 522
 523 #if 0
 524         iscsi_session_logout(is);
 525 #endif
 526         cv_signal(&is->is_maintenance_cv);
 527 }
 528
 529 static void
 530 iscsi_callout(void *context)
 531 {
 532         struct icl_pdu *request;
 533         struct iscsi_bhs_nop_out *bhsno;
 534         struct iscsi_session *is;
 535         bool reconnect_needed = false;
 536
 537         is = context;
 538
 539         if (is->is_terminating)
 540                 return;
 541
 542         callout_schedule(&is->is_callout, 1 * hz);
 543
 544         ISCSI_SESSION_LOCK(is);
 545         is->is_timeout++;
 546
 547         if (is->is_waiting_for_iscsid) {
 548                 if (iscsid_timeout > 0 && is->is_timeout > iscsid_timeout) {
 549                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 550                             "for %d seconds; reconnecting",
 551                             is->is_timeout);
 552                         reconnect_needed = true;
 553                 }
 554                 goto out;
 555         }
 556
 557         if (is->is_login_phase) {
 558                 if (login_timeout > 0 && is->is_timeout > login_timeout) {
 559                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 560                             "reconnecting", is->is_timeout);
 561                         reconnect_needed = true;
 562                 }
 563                 goto out;
 564         }
 565
 566         if (ping_timeout <= 0) {
 567                 /*
 568                  * Pings are disabled.  Don't send NOP-Out in this case.
 569                  * Reset the timeout, to avoid triggering reconnection,
 570                  * should the user decide to reenable them.
 571                  */
 572                 is->is_timeout = 0;
 573                 goto out;
 574         }
 575
 576         if (is->is_timeout >= ping_timeout) {
 577                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 578                     "reconnecting", ping_timeout);
 579                 reconnect_needed = true;
 580                 goto out;
 581         }
 582
 583         ISCSI_SESSION_UNLOCK(is);
 584
 585         /*
 586          * If the ping was reset less than one second ago - which means
 587          * that we've received some PDU during the last second - assume
 588          * the traffic flows correctly and don't bother sending a NOP-Out.
 589          *
 590          * (It's 2 - one for one second, and one for incrementing is_timeout
 591          * earlier in this routine.)
 592          */
 593         if (is->is_timeout < 2)
 594                 return;
 595
 596         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 597         if (request == NULL) {
 598                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 599                 return;
 600         }
 601         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 602         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 603             ISCSI_BHS_OPCODE_IMMEDIATE;
 604         bhsno->bhsno_flags = 0x80;
 605         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 606         iscsi_pdu_queue(request);
 607         return;
 608
 609 out:
 610         ISCSI_SESSION_UNLOCK(is);
 611
 612         if (reconnect_needed)
 613                 iscsi_session_reconnect(is);
 614 }
 615
 616 static void
 617 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 618 {
 619         const struct iscsi_bhs_data_in *bhsdi;
 620         struct iscsi_session *is;
 621         uint32_t expcmdsn, maxcmdsn, statsn;
 622
 623         is = PDU_SESSION(response);
 624
 625         ISCSI_SESSION_LOCK_ASSERT(is);
 626
 627         /*
 628          * We're only using fields common for all the response
 629          * (target -> initiator) PDUs.
 630          */
 631         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 632         /*
 633          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 634          * and Residual Count only have meaningful content if the S bit
 635          * is set to 1", so we also need to check the bit specific for
 636          * Data-In PDU.
 637          */
 638         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 639             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 640                 statsn = ntohl(bhsdi->bhsdi_statsn);
 641                 if (statsn != is->is_statsn && statsn != (is->is_statsn + 1)) {
 642                         /* XXX: This is normal situation for MCS */
 643                         ISCSI_SESSION_WARN(is, "PDU 0x%x StatSN %u != "
 644                             "session ExpStatSN %u (or + 1); reconnecting",
 645                             bhsdi->bhsdi_opcode, statsn, is->is_statsn);
 646                         iscsi_session_reconnect(is);
 647                 }
 648                 if (ISCSI_SNGT(statsn, is->is_statsn))
 649                         is->is_statsn = statsn;
 650         }
 651
 652         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 653         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 654
 655         if (ISCSI_SNLT(maxcmdsn + 1, expcmdsn)) {
 656                 ISCSI_SESSION_DEBUG(is,
 657                     "PDU MaxCmdSN %u + 1 < PDU ExpCmdSN %u; ignoring",
 658                     maxcmdsn, expcmdsn);
 659         } else {
 660                 if (ISCSI_SNGT(maxcmdsn, is->is_maxcmdsn)) {
 661                         is->is_maxcmdsn = maxcmdsn;
 662
 663                         /*
 664                          * Command window increased; kick the maintanance thread
 665                          * to send out postponed commands.
 666                          */
 667                         if (!STAILQ_EMPTY(&is->is_postponed))
 668                                 cv_signal(&is->is_maintenance_cv);
 669                 } else if (ISCSI_SNLT(maxcmdsn, is->is_maxcmdsn)) {
 670                         /* XXX: This is normal situation for MCS */
 671                         ISCSI_SESSION_DEBUG(is,
 672                             "PDU MaxCmdSN %u < session MaxCmdSN %u; ignoring",
 673                             maxcmdsn, is->is_maxcmdsn);
 674                 }
 675
 676                 if (ISCSI_SNGT(expcmdsn, is->is_expcmdsn)) {
 677                         is->is_expcmdsn = expcmdsn;
 678                 } else if (ISCSI_SNLT(expcmdsn, is->is_expcmdsn)) {
 679                         /* XXX: This is normal situation for MCS */
 680                         ISCSI_SESSION_DEBUG(is,
 681                             "PDU ExpCmdSN %u < session ExpCmdSN %u; ignoring",
 682                             expcmdsn, is->is_expcmdsn);
 683                 }
 684         }
 685
 686         /*
 687          * Every incoming PDU - not just NOP-In - resets the ping timer.
 688          * The purpose of the timeout is to reset the connection when it stalls;
 689          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 690          * in some queue.
 691          */
 692         is->is_timeout = 0;
 693 }
 694
 695 static void
 696 iscsi_receive_callback(struct icl_pdu *response)
 697 {
 698         struct iscsi_session *is;
 699
 700         is = PDU_SESSION(response);
 701
 702         ISCSI_SESSION_LOCK(is);
 703
 704 #ifdef ICL_KERNEL_PROXY
 705         if (is->is_login_phase) {
 706                 if (is->is_login_pdu == NULL)
 707                         is->is_login_pdu = response;
 708                 else
 709                         icl_pdu_free(response);
 710                 ISCSI_SESSION_UNLOCK(is);
 711                 cv_signal(&is->is_login_cv);
 712                 return;
 713         }
 714 #endif
 715
 716         iscsi_pdu_update_statsn(response);
 717
 718         /*
 719          * The handling routine is responsible for freeing the PDU
 720          * when it's no longer needed.
 721          */
 722         switch (response->ip_bhs->bhs_opcode) {
 723         case ISCSI_BHS_OPCODE_NOP_IN:
 724                 iscsi_pdu_handle_nop_in(response);
 725                 ISCSI_SESSION_UNLOCK(is);
 726                 break;
 727         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 728                 iscsi_pdu_handle_scsi_response(response);
 729                 /* Session lock dropped inside. */
 730                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 731                 break;
 732         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
 733                 iscsi_pdu_handle_task_response(response);
 734                 ISCSI_SESSION_UNLOCK(is);
 735                 break;
 736         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 737                 iscsi_pdu_handle_data_in(response);
 738                 /* Session lock dropped inside. */
 739                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 740                 break;
 741         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 742                 iscsi_pdu_handle_logout_response(response);
 743                 ISCSI_SESSION_UNLOCK(is);
 744                 break;
 745         case ISCSI_BHS_OPCODE_R2T:
 746                 iscsi_pdu_handle_r2t(response);
 747                 ISCSI_SESSION_UNLOCK(is);
 748                 break;
 749         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 750                 iscsi_pdu_handle_async_message(response);
 751                 ISCSI_SESSION_UNLOCK(is);
 752                 break;
 753         case ISCSI_BHS_OPCODE_REJECT:
 754                 iscsi_pdu_handle_reject(response);
 755                 ISCSI_SESSION_UNLOCK(is);
 756                 break;
 757         default:
 758                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 759                     "opcode 0x%x; reconnecting",
 760                     response->ip_bhs->bhs_opcode);
 761                 iscsi_session_reconnect(is);
 762                 ISCSI_SESSION_UNLOCK(is);
 763                 icl_pdu_free(response);
 764         }
 765 }
 766
 767 static void
 768 iscsi_error_callback(struct icl_conn *ic)
 769 {
 770         struct iscsi_session *is;
 771
 772         is = CONN_SESSION(ic);
 773
 774         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 775         iscsi_session_reconnect(is);
 776 }
 777
 778 static void
 779 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 780 {
 781         struct iscsi_session *is;
 782         struct iscsi_bhs_nop_out *bhsno;
 783         struct iscsi_bhs_nop_in *bhsni;
 784         struct icl_pdu *request;
 785         void *data = NULL;
 786         size_t datasize;
 787         int error;
 788
 789         is = PDU_SESSION(response);
 790         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 791
 792         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 793                 /*
 794                  * Nothing to do; iscsi_pdu_update_statsn() already
 795                  * zeroed the timeout.
 796                  */
 797                 icl_pdu_free(response);
 798                 return;
 799         }
 800
 801         datasize = icl_pdu_data_segment_length(response);
 802         if (datasize > 0) {
 803                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 804                 if (data == NULL) {
 805                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 806                             "reconnecting");
 807                         icl_pdu_free(response);
 808                         iscsi_session_reconnect(is);
 809                         return;
 810                 }
 811                 icl_pdu_get_data(response, 0, data, datasize);
 812         }
 813
 814         request = icl_pdu_new(response->ip_conn, M_NOWAIT);
 815         if (request == NULL) {
 816                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 817                     "reconnecting");
 818                 free(data, M_ISCSI);
 819                 icl_pdu_free(response);
 820                 iscsi_session_reconnect(is);
 821                 return;
 822         }
 823         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 824         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 825             ISCSI_BHS_OPCODE_IMMEDIATE;
 826         bhsno->bhsno_flags = 0x80;
 827         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 828         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 829         if (datasize > 0) {
 830                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 831                 if (error != 0) {
 832                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 833                             "reconnecting");
 834                         free(data, M_ISCSI);
 835                         icl_pdu_free(request);
 836                         icl_pdu_free(response);
 837                         iscsi_session_reconnect(is);
 838                         return;
 839                 }
 840                 free(data, M_ISCSI);
 841         }
 842
 843         icl_pdu_free(response);
 844         iscsi_pdu_queue_locked(request);
 845 }
 846
 847 static void
 848 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 849 {
 850         struct iscsi_bhs_scsi_response *bhssr;
 851         struct iscsi_outstanding *io;
 852         struct iscsi_session *is;
 853         union ccb *ccb;
 854         struct ccb_scsiio *csio;
 855         size_t data_segment_len, received;
 856         uint16_t sense_len;
 857
 858         is = PDU_SESSION(response);
 859
 860         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 861         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 862         if (io == NULL || io->io_ccb == NULL) {
 863                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 864                 icl_pdu_free(response);
 865                 iscsi_session_reconnect(is);
 866                 ISCSI_SESSION_UNLOCK(is);
 867                 return;
 868         }
 869
 870         ccb = io->io_ccb;
 871         received = io->io_received;
 872         iscsi_outstanding_remove(is, io);
 873         ISCSI_SESSION_UNLOCK(is);
 874
 875         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 876                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 877                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 878                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 879                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 880                 }
 881                 ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 882         } else if (bhssr->bhssr_status == 0) {
 883                 ccb->ccb_h.status = CAM_REQ_CMP;
 884         } else {
 885                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 886                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 887                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 888                 }
 889                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 890                 ccb->csio.scsi_status = bhssr->bhssr_status;
 891         }
 892
 893         csio = &ccb->csio;
 894         data_segment_len = icl_pdu_data_segment_length(response);
 895         if (data_segment_len > 0) {
 896                 if (data_segment_len < sizeof(sense_len)) {
 897                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 898                             data_segment_len);
 899                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 900                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 901                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 902                         }
 903                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 904                         goto out;
 905                 }
 906                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 907                 sense_len = ntohs(sense_len);
 908 #if 0
 909                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 910                     sense_len, data_segment_len);
 911 #endif
 912                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 913                         ISCSI_SESSION_WARN(is, "truncated data segment "
 914                             "(%zd bytes, should be %zd)",
 915                             data_segment_len, sizeof(sense_len) + sense_len);
 916                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 917                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 918                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 919                         }
 920                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 921                         goto out;
 922                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 923                         ISCSI_SESSION_WARN(is, "oversize data segment "
 924                             "(%zd bytes, should be %zd)",
 925                             data_segment_len, sizeof(sense_len) + sense_len);
 926                 if (sense_len > csio->sense_len) {
 927                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 928                             sense_len, csio->sense_len);
 929                         sense_len = csio->sense_len;
 930                 }
 931                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 932                 csio->sense_resid = csio->sense_len - sense_len;
 933                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 934         }
 935
 936 out:
 937         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 938                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 939
 940         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 941                 KASSERT(received <= csio->dxfer_len,
 942                     ("received > csio->dxfer_len"));
 943                 if (received < csio->dxfer_len) {
 944                         if (csio->resid != csio->dxfer_len - received) {
 945                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 946                                     "target indicates %d, we calculated %zd",
 947                                     csio->resid, csio->dxfer_len - received);
 948                         }
 949                         csio->resid = csio->dxfer_len - received;
 950                 }
 951         }
 952
 953         xpt_done(ccb);
 954         icl_pdu_free(response);
 955 }
 956
 957 static void
 958 iscsi_pdu_handle_task_response(struct icl_pdu *response)
 959 {
 960         struct iscsi_bhs_task_management_response *bhstmr;
 961         struct iscsi_outstanding *io, *aio;
 962         struct iscsi_session *is;
 963
 964         is = PDU_SESSION(response);
 965
 966         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
 967         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
 968         if (io == NULL || io->io_ccb != NULL) {
 969                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
 970                     bhstmr->bhstmr_initiator_task_tag);
 971                 icl_pdu_free(response);
 972                 iscsi_session_reconnect(is);
 973                 return;
 974         }
 975
 976         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
 977                 ISCSI_SESSION_WARN(is, "task response 0x%x",
 978                     bhstmr->bhstmr_response);
 979         } else {
 980                 aio = iscsi_outstanding_find(is, io->io_datasn);
 981                 if (aio != NULL && aio->io_ccb != NULL)
 982                         iscsi_session_terminate_task(is, aio, false);
 983         }
 984
 985         iscsi_outstanding_remove(is, io);
 986         icl_pdu_free(response);
 987 }
 988
 989 static void
 990 iscsi_pdu_handle_data_in(struct icl_pdu *response)
 991 {
 992         struct iscsi_bhs_data_in *bhsdi;
 993         struct iscsi_outstanding *io;
 994         struct iscsi_session *is;
 995         union ccb *ccb;
 996         struct ccb_scsiio *csio;
 997         size_t data_segment_len, received, oreceived;
 998
 999         is = PDU_SESSION(response);
1000         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
1001         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
1002         if (io == NULL || io->io_ccb == NULL) {
1003                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
1004                 icl_pdu_free(response);
1005                 iscsi_session_reconnect(is);
1006                 ISCSI_SESSION_UNLOCK(is);
1007                 return;
1008         }
1009
1010         data_segment_len = icl_pdu_data_segment_length(response);
1011         if (data_segment_len == 0) {
1012                 /*
1013                  * "The sending of 0 length data segments should be avoided,
1014                  * but initiators and targets MUST be able to properly receive
1015                  * 0 length data segments."
1016                  */
1017                 ISCSI_SESSION_UNLOCK(is);
1018                 icl_pdu_free(response);
1019                 return;
1020         }
1021
1022         /*
1023          * We need to track this for security reasons - without it, malicious target
1024          * could respond to SCSI READ without sending Data-In PDUs, which would result
1025          * in read operation on the initiator side returning random kernel data.
1026          */
1027         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1028                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1029                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1030                 icl_pdu_free(response);
1031                 iscsi_session_reconnect(is);
1032                 ISCSI_SESSION_UNLOCK(is);
1033                 return;
1034         }
1035
1036         ccb = io->io_ccb;
1037         csio = &ccb->csio;
1038
1039         if (io->io_received + data_segment_len > csio->dxfer_len) {
1040                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1041                     "at offset %zd, buffer is %d)",
1042                     data_segment_len, io->io_received, csio->dxfer_len);
1043                 icl_pdu_free(response);
1044                 iscsi_session_reconnect(is);
1045                 ISCSI_SESSION_UNLOCK(is);
1046                 return;
1047         }
1048
1049         oreceived = io->io_received;
1050         io->io_received += data_segment_len;
1051         received = io->io_received;
1052         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0)
1053                 iscsi_outstanding_remove(is, io);
1054         ISCSI_SESSION_UNLOCK(is);
1055
1056         icl_pdu_get_data(response, 0, csio->data_ptr + oreceived, data_segment_len);
1057
1058         /*
1059          * XXX: Check DataSN.
1060          * XXX: Check F.
1061          */
1062         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1063                 /*
1064                  * Nothing more to do.
1065                  */
1066                 icl_pdu_free(response);
1067                 return;
1068         }
1069
1070         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1071         if (bhsdi->bhsdi_status == 0) {
1072                 ccb->ccb_h.status = CAM_REQ_CMP;
1073         } else {
1074                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1075                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1076                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1077                 }
1078                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1079                 csio->scsi_status = bhsdi->bhsdi_status;
1080         }
1081
1082         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1083                 KASSERT(received <= csio->dxfer_len,
1084                     ("received > csio->dxfer_len"));
1085                 if (received < csio->dxfer_len) {
1086                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1087                         if (csio->resid != csio->dxfer_len - received) {
1088                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1089                                     "target indicates %d, we calculated %zd",
1090                                     csio->resid, csio->dxfer_len - received);
1091                         }
1092                         csio->resid = csio->dxfer_len - received;
1093                 }
1094         }
1095
1096         xpt_done(ccb);
1097         icl_pdu_free(response);
1098 }
1099
1100 static void
1101 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1102 {
1103
1104         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1105         icl_pdu_free(response);
1106 }
1107
1108 static void
1109 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1110 {
1111         struct icl_pdu *request;
1112         struct iscsi_session *is;
1113         struct iscsi_bhs_r2t *bhsr2t;
1114         struct iscsi_bhs_data_out *bhsdo;
1115         struct iscsi_outstanding *io;
1116         struct ccb_scsiio *csio;
1117         size_t off, len, total_len;
1118         int error;
1119
1120         is = PDU_SESSION(response);
1121
1122         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1123         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1124         if (io == NULL || io->io_ccb == NULL) {
1125                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1126                     bhsr2t->bhsr2t_initiator_task_tag);
1127                 icl_pdu_free(response);
1128                 iscsi_session_reconnect(is);
1129                 return;
1130         }
1131
1132         csio = &io->io_ccb->csio;
1133
1134         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1135                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1136                 icl_pdu_free(response);
1137                 iscsi_session_reconnect(is);
1138                 return;
1139         }
1140
1141         /*
1142          * XXX: Verify R2TSN.
1143          */
1144
1145         io->io_datasn = 0;
1146
1147         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1148         if (off > csio->dxfer_len) {
1149                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1150                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1151                 icl_pdu_free(response);
1152                 iscsi_session_reconnect(is);
1153                 return;
1154         }
1155
1156         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1157         if (total_len == 0 || total_len > csio->dxfer_len) {
1158                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1159                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1160                 icl_pdu_free(response);
1161                 iscsi_session_reconnect(is);
1162                 return;
1163         }
1164
1165         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1166
1167         for (;;) {
1168                 len = total_len;
1169
1170                 if (len > is->is_max_data_segment_length)
1171                         len = is->is_max_data_segment_length;
1172
1173                 if (off + len > csio->dxfer_len) {
1174                         ISCSI_SESSION_WARN(is, "target requested invalid "
1175                             "length/offset %zd, buffer is %d; reconnecting",
1176                             off + len, csio->dxfer_len);
1177                         icl_pdu_free(response);
1178                         iscsi_session_reconnect(is);
1179                         return;
1180                 }
1181
1182                 request = icl_pdu_new(response->ip_conn, M_NOWAIT);
1183                 if (request == NULL) {
1184                         icl_pdu_free(response);
1185                         iscsi_session_reconnect(is);
1186                         return;
1187                 }
1188
1189                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1190                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1191                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1192                 bhsdo->bhsdo_initiator_task_tag =
1193                     bhsr2t->bhsr2t_initiator_task_tag;
1194                 bhsdo->bhsdo_target_transfer_tag =
1195                     bhsr2t->bhsr2t_target_transfer_tag;
1196                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1197                 bhsdo->bhsdo_buffer_offset = htonl(off);
1198                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1199                     M_NOWAIT);
1200                 if (error != 0) {
1201                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1202                             "reconnecting");
1203                         icl_pdu_free(request);
1204                         icl_pdu_free(response);
1205                         iscsi_session_reconnect(is);
1206                         return;
1207                 }
1208
1209                 off += len;
1210                 total_len -= len;
1211
1212                 if (total_len == 0) {
1213                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1214                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1215                 } else {
1216                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1217                 }
1218
1219                 iscsi_pdu_queue_locked(request);
1220
1221                 if (total_len == 0)
1222                         break;
1223         }
1224
1225         icl_pdu_free(response);
1226 }
1227
1228 static void
1229 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1230 {
1231         struct iscsi_bhs_asynchronous_message *bhsam;
1232         struct iscsi_session *is;
1233
1234         is = PDU_SESSION(response);
1235         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1236         switch (bhsam->bhsam_async_event) {
1237         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1238                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1239                 iscsi_session_logout(is);
1240                 iscsi_session_terminate(is);
1241                 break;
1242         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1243                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1244                 break;
1245         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1246                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1247                 break;
1248         default:
1249                 /*
1250                  * XXX: Technically, we're obligated to also handle
1251                  *      parameter renegotiation.
1252                  */
1253                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1254                 break;
1255         }
1256
1257         icl_pdu_free(response);
1258 }
1259
1260 static void
1261 iscsi_pdu_handle_reject(struct icl_pdu *response)
1262 {
1263         struct iscsi_bhs_reject *bhsr;
1264         struct iscsi_session *is;
1265
1266         is = PDU_SESSION(response);
1267         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1268         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1269             bhsr->bhsr_reason);
1270
1271         icl_pdu_free(response);
1272 }
1273
1274 static int
1275 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1276     struct iscsi_daemon_request *request)
1277 {
1278         struct iscsi_session *is;
1279         int error;
1280
1281         sx_slock(&sc->sc_lock);
1282         for (;;) {
1283                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1284                         ISCSI_SESSION_LOCK(is);
1285                         if (is->is_waiting_for_iscsid)
1286                                 break;
1287                         ISCSI_SESSION_UNLOCK(is);
1288                 }
1289
1290                 if (is == NULL) {
1291                         /*
1292                          * No session requires attention from iscsid(8); wait.
1293                          */
1294                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1295                         if (error != 0) {
1296                                 sx_sunlock(&sc->sc_lock);
1297                                 return (error);
1298                         }
1299                         continue;
1300                 }
1301
1302                 is->is_waiting_for_iscsid = false;
1303                 is->is_login_phase = true;
1304                 is->is_reason[0] = '\0';
1305                 ISCSI_SESSION_UNLOCK(is);
1306
1307                 request->idr_session_id = is->is_id;
1308                 memcpy(&request->idr_isid, &is->is_isid,
1309                     sizeof(request->idr_isid));
1310                 request->idr_tsih = 0;  /* New or reinstated session. */
1311                 memcpy(&request->idr_conf, &is->is_conf,
1312                     sizeof(request->idr_conf));
1313
1314                 sx_sunlock(&sc->sc_lock);
1315                 return (0);
1316         }
1317 }
1318
1319 static int
1320 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1321     struct iscsi_daemon_handoff *handoff)
1322 {
1323         struct iscsi_session *is;
1324         int error;
1325
1326         sx_slock(&sc->sc_lock);
1327
1328         /*
1329          * Find the session to hand off socket to.
1330          */
1331         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1332                 if (is->is_id == handoff->idh_session_id)
1333                         break;
1334         }
1335         if (is == NULL) {
1336                 sx_sunlock(&sc->sc_lock);
1337                 return (ESRCH);
1338         }
1339         ISCSI_SESSION_LOCK(is);
1340         if (is->is_conf.isc_discovery || is->is_terminating) {
1341                 ISCSI_SESSION_UNLOCK(is);
1342                 sx_sunlock(&sc->sc_lock);
1343                 return (EINVAL);
1344         }
1345         if (is->is_connected) {
1346                 /*
1347                  * This might have happened because another iscsid(8)
1348                  * instance handed off the connection in the meantime.
1349                  * Just return.
1350                  */
1351                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1352                     "session");
1353                 ISCSI_SESSION_UNLOCK(is);
1354                 sx_sunlock(&sc->sc_lock);
1355                 return (EBUSY);
1356         }
1357
1358         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1359             sizeof(is->is_target_alias));
1360         is->is_tsih = handoff->idh_tsih;
1361         is->is_statsn = handoff->idh_statsn;
1362         is->is_initial_r2t = handoff->idh_initial_r2t;
1363         is->is_immediate_data = handoff->idh_immediate_data;
1364         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1365         is->is_max_burst_length = handoff->idh_max_burst_length;
1366         is->is_first_burst_length = handoff->idh_first_burst_length;
1367
1368         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1369                 is->is_conn->ic_header_crc32c = true;
1370         else
1371                 is->is_conn->ic_header_crc32c = false;
1372         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1373                 is->is_conn->ic_data_crc32c = true;
1374         else
1375                 is->is_conn->ic_data_crc32c = false;
1376
1377         is->is_cmdsn = 0;
1378         is->is_expcmdsn = 0;
1379         is->is_maxcmdsn = 0;
1380         is->is_waiting_for_iscsid = false;
1381         is->is_login_phase = false;
1382         is->is_timeout = 0;
1383         is->is_connected = true;
1384         is->is_reason[0] = '\0';
1385
1386         ISCSI_SESSION_UNLOCK(is);
1387
1388 #ifdef ICL_KERNEL_PROXY
1389         if (handoff->idh_socket != 0) {
1390 #endif
1391                 /*
1392                  * Handoff without using ICL proxy.
1393                  */
1394                 error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1395                 if (error != 0) {
1396                         sx_sunlock(&sc->sc_lock);
1397                         iscsi_session_terminate(is);
1398                         return (error);
1399                 }
1400 #ifdef ICL_KERNEL_PROXY
1401         }
1402 #endif
1403
1404         sx_sunlock(&sc->sc_lock);
1405
1406         if (is->is_sim != NULL) {
1407                 /*
1408                  * When reconnecting, there already is SIM allocated for the session.
1409                  */
1410                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1411                 ISCSI_SESSION_LOCK(is);
1412                 ISCSI_SESSION_DEBUG(is, "releasing");
1413                 xpt_release_simq(is->is_sim, 1);
1414                 is->is_simq_frozen = false;
1415                 ISCSI_SESSION_UNLOCK(is);
1416
1417         } else {
1418                 ISCSI_SESSION_LOCK(is);
1419                 is->is_devq = cam_simq_alloc(maxtags);
1420                 if (is->is_devq == NULL) {
1421                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1422                         iscsi_session_terminate(is);
1423                         return (ENOMEM);
1424                 }
1425
1426                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1427                     is, is->is_id /* unit */, &is->is_lock,
1428                     1, maxtags, is->is_devq);
1429                 if (is->is_sim == NULL) {
1430                         ISCSI_SESSION_UNLOCK(is);
1431                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1432                         cam_simq_free(is->is_devq);
1433                         iscsi_session_terminate(is);
1434                         return (ENOMEM);
1435                 }
1436
1437                 error = xpt_bus_register(is->is_sim, NULL, 0);
1438                 if (error != 0) {
1439                         ISCSI_SESSION_UNLOCK(is);
1440                         ISCSI_SESSION_WARN(is, "failed to register bus");
1441                         iscsi_session_terminate(is);
1442                         return (ENOMEM);
1443                 }
1444
1445                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1446                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1447                     CAM_LUN_WILDCARD);
1448                 if (error != CAM_REQ_CMP) {
1449                         ISCSI_SESSION_UNLOCK(is);
1450                         ISCSI_SESSION_WARN(is, "failed to create path");
1451                         iscsi_session_terminate(is);
1452                         return (ENOMEM);
1453                 }
1454                 ISCSI_SESSION_UNLOCK(is);
1455         }
1456
1457         return (0);
1458 }
1459
1460 static int
1461 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1462     struct iscsi_daemon_fail *fail)
1463 {
1464         struct iscsi_session *is;
1465
1466         sx_slock(&sc->sc_lock);
1467
1468         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1469                 if (is->is_id == fail->idf_session_id)
1470                         break;
1471         }
1472         if (is == NULL) {
1473                 sx_sunlock(&sc->sc_lock);
1474                 return (ESRCH);
1475         }
1476         ISCSI_SESSION_LOCK(is);
1477         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1478             fail->idf_reason);
1479         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1480         //is->is_waiting_for_iscsid = false;
1481         //is->is_login_phase = true;
1482         //iscsi_session_reconnect(is);
1483         ISCSI_SESSION_UNLOCK(is);
1484         sx_sunlock(&sc->sc_lock);
1485
1486         return (0);
1487 }
1488
1489 #ifdef ICL_KERNEL_PROXY
1490 static int
1491 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1492     struct iscsi_daemon_connect *idc)
1493 {
1494         struct iscsi_session *is;
1495         struct sockaddr *from_sa, *to_sa;
1496         int error;
1497
1498         sx_slock(&sc->sc_lock);
1499         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1500                 if (is->is_id == idc->idc_session_id)
1501                         break;
1502         }
1503         if (is == NULL) {
1504                 sx_sunlock(&sc->sc_lock);
1505                 return (ESRCH);
1506         }
1507         sx_sunlock(&sc->sc_lock);
1508
1509         if (idc->idc_from_addrlen > 0) {
1510                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1511                 if (error != 0) {
1512                         ISCSI_SESSION_WARN(is,
1513                             "getsockaddr failed with error %d", error);
1514                         return (error);
1515                 }
1516         } else {
1517                 from_sa = NULL;
1518         }
1519         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1520         if (error != 0) {
1521                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1522                     error);
1523                 free(from_sa, M_SONAME);
1524                 return (error);
1525         }
1526
1527         ISCSI_SESSION_LOCK(is);
1528         is->is_waiting_for_iscsid = false;
1529         is->is_login_phase = true;
1530         is->is_timeout = 0;
1531         ISCSI_SESSION_UNLOCK(is);
1532
1533         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1534             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1535         free(from_sa, M_SONAME);
1536         free(to_sa, M_SONAME);
1537
1538         /*
1539          * Digests are always disabled during login phase.
1540          */
1541         is->is_conn->ic_header_crc32c = false;
1542         is->is_conn->ic_data_crc32c = false;
1543
1544         return (error);
1545 }
1546
1547 static int
1548 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1549     struct iscsi_daemon_send *ids)
1550 {
1551         struct iscsi_session *is;
1552         struct icl_pdu *ip;
1553         size_t datalen;
1554         void *data;
1555         int error;
1556
1557         sx_slock(&sc->sc_lock);
1558         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1559                 if (is->is_id == ids->ids_session_id)
1560                         break;
1561         }
1562         if (is == NULL) {
1563                 sx_sunlock(&sc->sc_lock);
1564                 return (ESRCH);
1565         }
1566         sx_sunlock(&sc->sc_lock);
1567
1568         if (is->is_login_phase == false)
1569                 return (EBUSY);
1570
1571         if (is->is_terminating || is->is_reconnecting)
1572                 return (EIO);
1573
1574         datalen = ids->ids_data_segment_len;
1575         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1576                 return (EINVAL);
1577         if (datalen > 0) {
1578                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1579                 error = copyin(ids->ids_data_segment, data, datalen);
1580                 if (error != 0) {
1581                         free(data, M_ISCSI);
1582                         return (error);
1583                 }
1584         }
1585
1586         ip = icl_pdu_new(is->is_conn, M_WAITOK);
1587         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1588         if (datalen > 0) {
1589                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1590                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1591                 free(data, M_ISCSI);
1592         }
1593         icl_pdu_queue(ip);
1594
1595         return (0);
1596 }
1597
1598 static int
1599 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1600     struct iscsi_daemon_receive *idr)
1601 {
1602         struct iscsi_session *is;
1603         struct icl_pdu *ip;
1604         void *data;
1605
1606         sx_slock(&sc->sc_lock);
1607         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1608                 if (is->is_id == idr->idr_session_id)
1609                         break;
1610         }
1611         if (is == NULL) {
1612                 sx_sunlock(&sc->sc_lock);
1613                 return (ESRCH);
1614         }
1615         sx_sunlock(&sc->sc_lock);
1616
1617         if (is->is_login_phase == false)
1618                 return (EBUSY);
1619
1620         ISCSI_SESSION_LOCK(is);
1621         while (is->is_login_pdu == NULL &&
1622             is->is_terminating == false &&
1623             is->is_reconnecting == false)
1624                 cv_wait(&is->is_login_cv, &is->is_lock);
1625         if (is->is_terminating || is->is_reconnecting) {
1626                 ISCSI_SESSION_UNLOCK(is);
1627                 return (EIO);
1628         }
1629         ip = is->is_login_pdu;
1630         is->is_login_pdu = NULL;
1631         ISCSI_SESSION_UNLOCK(is);
1632
1633         if (ip->ip_data_len > idr->idr_data_segment_len) {
1634                 icl_pdu_free(ip);
1635                 return (EMSGSIZE);
1636         }
1637
1638         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1639         if (ip->ip_data_len > 0) {
1640                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1641                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1642                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1643                 free(data, M_ISCSI);
1644         }
1645
1646         icl_pdu_free(ip);
1647
1648         return (0);
1649 }
1650 #endif /* ICL_KERNEL_PROXY */
1651
1652 static void
1653 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1654 {
1655         /*
1656          * Just make sure all the fields are null-terminated.
1657          *
1658          * XXX: This is not particularly secure.  We should
1659          *      create our own conf and then copy in relevant
1660          *      fields.
1661          */
1662         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1663         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1664         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1665         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1666         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1667         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1668         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1669         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1670         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1671 }
1672
1673 static bool
1674 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1675 {
1676
1677         if (isc->isc_initiator[0] == '\0') {
1678                 ISCSI_DEBUG("empty isc_initiator");
1679                 return (false);
1680         }
1681
1682         if (isc->isc_target_addr[0] == '\0') {
1683                 ISCSI_DEBUG("empty isc_target_addr");
1684                 return (false);
1685         }
1686
1687         if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1688                 ISCSI_DEBUG("non-empty isc_target for discovery session");
1689                 return (false);
1690         }
1691
1692         if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1693                 ISCSI_DEBUG("empty isc_target for non-discovery session");
1694                 return (false);
1695         }
1696
1697         return (true);
1698 }
1699
1700 static int
1701 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1702 {
1703         struct iscsi_session *is;
1704         const struct iscsi_session *is2;
1705         int error;
1706
1707         iscsi_sanitize_session_conf(&isa->isa_conf);
1708         if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1709                 return (EINVAL);
1710
1711         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1712         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1713
1714         sx_xlock(&sc->sc_lock);
1715
1716         /*
1717          * Prevent duplicates.
1718          */
1719         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1720                 if (!!is->is_conf.isc_discovery !=
1721                     !!is2->is_conf.isc_discovery)
1722                         continue;
1723
1724                 if (strcmp(is->is_conf.isc_target_addr,
1725                     is2->is_conf.isc_target_addr) != 0)
1726                         continue;
1727
1728                 if (is->is_conf.isc_discovery == 0 &&
1729                     strcmp(is->is_conf.isc_target,
1730                     is2->is_conf.isc_target) != 0)
1731                         continue;
1732
1733                 sx_xunlock(&sc->sc_lock);
1734                 free(is, M_ISCSI);
1735                 return (EBUSY);
1736         }
1737
1738         is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1739         is->is_conn->ic_receive = iscsi_receive_callback;
1740         is->is_conn->ic_error = iscsi_error_callback;
1741         is->is_conn->ic_prv0 = is;
1742         TAILQ_INIT(&is->is_outstanding);
1743         STAILQ_INIT(&is->is_postponed);
1744         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1745         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1746 #ifdef ICL_KERNEL_PROXY
1747         cv_init(&is->is_login_cv, "iscsi_login");
1748 #endif
1749
1750         is->is_softc = sc;
1751         sc->sc_last_session_id++;
1752         is->is_id = sc->sc_last_session_id;
1753         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1754         arc4rand(&is->is_isid[1], 5, 0);
1755         is->is_tsih = 0;
1756         callout_init(&is->is_callout, 1);
1757         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1758         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1759
1760         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1761         if (error != 0) {
1762                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1763                 return (error);
1764         }
1765
1766         /*
1767          * Trigger immediate reconnection.
1768          */
1769         ISCSI_SESSION_LOCK(is);
1770         is->is_waiting_for_iscsid = true;
1771         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1772         ISCSI_SESSION_UNLOCK(is);
1773         cv_signal(&sc->sc_cv);
1774
1775         sx_xunlock(&sc->sc_lock);
1776
1777         return (0);
1778 }
1779
1780 static bool
1781 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1782     unsigned int id2, const struct iscsi_session_conf *c2)
1783 {
1784
1785         if (id2 != 0 && id2 != id1)
1786                 return (false);
1787         if (c2->isc_target[0] != '\0' &&
1788             strcmp(c1->isc_target, c2->isc_target) != 0)
1789                 return (false);
1790         if (c2->isc_target_addr[0] != '\0' &&
1791             strcmp(c1->isc_target_addr, c2->isc_target_addr) != 0)
1792                 return (false);
1793         return (true);
1794 }
1795
1796 static int
1797 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1798     struct iscsi_session_remove *isr)
1799 {
1800         struct iscsi_session *is, *tmp;
1801         bool found = false;
1802
1803         iscsi_sanitize_session_conf(&isr->isr_conf);
1804
1805         sx_xlock(&sc->sc_lock);
1806         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1807                 ISCSI_SESSION_LOCK(is);
1808                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1809                     isr->isr_session_id, &isr->isr_conf)) {
1810                         found = true;
1811                         iscsi_session_logout(is);
1812                         iscsi_session_terminate(is);
1813                 }
1814                 ISCSI_SESSION_UNLOCK(is);
1815         }
1816         sx_xunlock(&sc->sc_lock);
1817
1818         if (!found)
1819                 return (ESRCH);
1820
1821         return (0);
1822 }
1823
1824 static int
1825 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1826 {
1827         int error;
1828         unsigned int i = 0;
1829         struct iscsi_session *is;
1830         struct iscsi_session_state iss;
1831
1832         sx_slock(&sc->sc_lock);
1833         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1834                 if (i >= isl->isl_nentries) {
1835                         sx_sunlock(&sc->sc_lock);
1836                         return (EMSGSIZE);
1837                 }
1838                 memset(&iss, 0, sizeof(iss));
1839                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1840                 iss.iss_id = is->is_id;
1841                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1842                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1843
1844                 if (is->is_conn->ic_header_crc32c)
1845                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1846                 else
1847                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1848
1849                 if (is->is_conn->ic_data_crc32c)
1850                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1851                 else
1852                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1853
1854                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1855                 iss.iss_immediate_data = is->is_immediate_data;
1856                 iss.iss_connected = is->is_connected;
1857
1858                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1859                 if (error != 0) {
1860                         sx_sunlock(&sc->sc_lock);
1861                         return (error);
1862                 }
1863                 i++;
1864         }
1865         sx_sunlock(&sc->sc_lock);
1866
1867         isl->isl_nentries = i;
1868
1869         return (0);
1870 }
1871
1872 static int
1873 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
1874     struct iscsi_session_modify *ism)
1875 {
1876         struct iscsi_session *is;
1877
1878         iscsi_sanitize_session_conf(&ism->ism_conf);
1879         if (iscsi_valid_session_conf(&ism->ism_conf) == false)
1880                 return (EINVAL);
1881
1882         sx_xlock(&sc->sc_lock);
1883         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1884                 ISCSI_SESSION_LOCK(is);
1885                 if (is->is_id == ism->ism_session_id)
1886                         break;
1887                 ISCSI_SESSION_UNLOCK(is);
1888         }
1889         if (is == NULL) {
1890                 sx_xunlock(&sc->sc_lock);
1891                 return (ESRCH);
1892         }
1893         sx_xunlock(&sc->sc_lock);
1894
1895         memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
1896         ISCSI_SESSION_UNLOCK(is);
1897
1898         iscsi_session_reconnect(is);
1899
1900         return (0);
1901 }
1902
1903 static int
1904 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1905     struct thread *td)
1906 {
1907         struct iscsi_softc *sc;
1908
1909         sc = dev->si_drv1;
1910
1911         switch (cmd) {
1912         case ISCSIDWAIT:
1913                 return (iscsi_ioctl_daemon_wait(sc,
1914                     (struct iscsi_daemon_request *)arg));
1915         case ISCSIDHANDOFF:
1916                 return (iscsi_ioctl_daemon_handoff(sc,
1917                     (struct iscsi_daemon_handoff *)arg));
1918         case ISCSIDFAIL:
1919                 return (iscsi_ioctl_daemon_fail(sc,
1920                     (struct iscsi_daemon_fail *)arg));
1921 #ifdef ICL_KERNEL_PROXY
1922         case ISCSIDCONNECT:
1923                 return (iscsi_ioctl_daemon_connect(sc,
1924                     (struct iscsi_daemon_connect *)arg));
1925         case ISCSIDSEND:
1926                 return (iscsi_ioctl_daemon_send(sc,
1927                     (struct iscsi_daemon_send *)arg));
1928         case ISCSIDRECEIVE:
1929                 return (iscsi_ioctl_daemon_receive(sc,
1930                     (struct iscsi_daemon_receive *)arg));
1931 #endif /* ICL_KERNEL_PROXY */
1932         case ISCSISADD:
1933                 return (iscsi_ioctl_session_add(sc,
1934                     (struct iscsi_session_add *)arg));
1935         case ISCSISREMOVE:
1936                 return (iscsi_ioctl_session_remove(sc,
1937                     (struct iscsi_session_remove *)arg));
1938         case ISCSISLIST:
1939                 return (iscsi_ioctl_session_list(sc,
1940                     (struct iscsi_session_list *)arg));
1941         case ISCSISMODIFY:
1942                 return (iscsi_ioctl_session_modify(sc,
1943                     (struct iscsi_session_modify *)arg));
1944         default:
1945                 return (EINVAL);
1946         }
1947 }
1948
1949 static uint64_t
1950 iscsi_encode_lun(uint32_t lun)
1951 {
1952         uint8_t encoded[8];
1953         uint64_t result;
1954
1955         memset(encoded, 0, sizeof(encoded));
1956
1957         if (lun < 256) {
1958                 /*
1959                  * Peripheral device addressing.
1960                  */
1961                 encoded[1] = lun;
1962         } else if (lun < 16384) {
1963                 /*
1964                  * Flat space addressing.
1965                  */
1966                 encoded[0] = 0x40;
1967                 encoded[0] |= (lun >> 8) & 0x3f;
1968                 encoded[1] = lun & 0xff;
1969         } else {
1970                 /*
1971                  * Extended flat space addressing.
1972                  */
1973                 encoded[0] = 0xd2;
1974                 encoded[1] = lun >> 16;
1975                 encoded[2] = lun >> 8;
1976                 encoded[3] = lun;
1977         }
1978
1979         memcpy(&result, encoded, sizeof(result));
1980         return (result);
1981 }
1982
1983 static struct iscsi_outstanding *
1984 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1985 {
1986         struct iscsi_outstanding *io;
1987
1988         ISCSI_SESSION_LOCK_ASSERT(is);
1989
1990         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1991                 if (io->io_initiator_task_tag == initiator_task_tag)
1992                         return (io);
1993         }
1994         return (NULL);
1995 }
1996
1997 static struct iscsi_outstanding *
1998 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
1999 {
2000         struct iscsi_outstanding *io;
2001
2002         ISCSI_SESSION_LOCK_ASSERT(is);
2003
2004         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2005                 if (io->io_ccb == ccb)
2006                         return (io);
2007         }
2008         return (NULL);
2009 }
2010
2011 static struct iscsi_outstanding *
2012 iscsi_outstanding_add(struct iscsi_session *is,
2013     uint32_t initiator_task_tag, union ccb *ccb)
2014 {
2015         struct iscsi_outstanding *io;
2016
2017         ISCSI_SESSION_LOCK_ASSERT(is);
2018
2019         KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
2020             ("initiator_task_tag 0x%x already added", initiator_task_tag));
2021
2022         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
2023         if (io == NULL) {
2024                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
2025                 return (NULL);
2026         }
2027         io->io_initiator_task_tag = initiator_task_tag;
2028         io->io_ccb = ccb;
2029         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
2030         return (io);
2031 }
2032
2033 static void
2034 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
2035 {
2036
2037         ISCSI_SESSION_LOCK_ASSERT(is);
2038
2039         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
2040         uma_zfree(iscsi_outstanding_zone, io);
2041 }
2042
2043 static void
2044 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
2045 {
2046         struct icl_pdu *request;
2047         struct iscsi_bhs_task_management_request *bhstmr;
2048         struct ccb_abort *cab = &ccb->cab;
2049         struct iscsi_outstanding *io, *aio;
2050
2051         ISCSI_SESSION_LOCK_ASSERT(is);
2052
2053 #if 0
2054         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2055 #else
2056         if (is->is_login_phase) {
2057                 ccb->ccb_h.status = CAM_REQ_ABORTED;
2058                 xpt_done(ccb);
2059                 return;
2060         }
2061 #endif
2062
2063         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
2064         if (aio == NULL) {
2065                 ccb->ccb_h.status = CAM_REQ_CMP;
2066                 xpt_done(ccb);
2067                 return;
2068         }
2069
2070         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2071         if (request == NULL) {
2072                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2073                 xpt_done(ccb);
2074                 return;
2075         }
2076
2077         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2078         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
2079         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
2080
2081         bhstmr->bhstmr_lun = iscsi_encode_lun(ccb->ccb_h.target_lun);
2082         bhstmr->bhstmr_initiator_task_tag = is->is_initiator_task_tag;
2083         is->is_initiator_task_tag++;
2084         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2085
2086         io = iscsi_outstanding_add(is, bhstmr->bhstmr_initiator_task_tag, NULL);
2087         if (io == NULL) {
2088                 icl_pdu_free(request);
2089                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2090                 xpt_done(ccb);
2091                 return;
2092         }
2093         io->io_datasn = aio->io_initiator_task_tag;
2094         iscsi_pdu_queue_locked(request);
2095 }
2096
2097 static void
2098 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2099 {
2100         struct icl_pdu *request;
2101         struct iscsi_bhs_scsi_command *bhssc;
2102         struct ccb_scsiio *csio;
2103         struct iscsi_outstanding *io;
2104         size_t len;
2105         int error;
2106
2107         ISCSI_SESSION_LOCK_ASSERT(is);
2108
2109 #if 0
2110         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2111 #else
2112         if (is->is_login_phase) {
2113                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2114                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2115                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2116                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2117                 }
2118                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2119                 xpt_done(ccb);
2120                 return;
2121         }
2122 #endif
2123
2124         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2125         if (request == NULL) {
2126                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2127                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2128                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2129                 }
2130                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2131                 xpt_done(ccb);
2132                 return;
2133         }
2134
2135         csio = &ccb->csio;
2136         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2137         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2138         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2139         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2140         case CAM_DIR_IN:
2141                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2142                 break;
2143         case CAM_DIR_OUT:
2144                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2145                 break;
2146         }
2147
2148         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2149                 switch (csio->tag_action) {
2150                 case MSG_HEAD_OF_Q_TAG:
2151                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2152                         break;
2153                 case MSG_ORDERED_Q_TAG:
2154                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2155                         break;
2156                 case MSG_ACA_TASK:
2157                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2158                         break;
2159                 case MSG_SIMPLE_Q_TAG:
2160                 default:
2161                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2162                         break;
2163                 }
2164         } else
2165                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2166
2167         bhssc->bhssc_lun = iscsi_encode_lun(csio->ccb_h.target_lun);
2168         bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
2169         is->is_initiator_task_tag++;
2170         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2171         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2172             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2173
2174         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2175                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2176         else
2177                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2178
2179         io = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
2180         if (io == NULL) {
2181                 icl_pdu_free(request);
2182                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2183                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2184                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2185                 }
2186                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2187                 xpt_done(ccb);
2188                 return;
2189         }
2190
2191         if (is->is_immediate_data &&
2192             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2193                 len = csio->dxfer_len;
2194                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2195                 if (len > is->is_first_burst_length) {
2196                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
2197                         len = is->is_first_burst_length;
2198                 }
2199                 if (len > is->is_max_data_segment_length) {
2200                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_max_data_segment_length);
2201                         len = is->is_max_data_segment_length;
2202                 }
2203
2204                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2205                 if (error != 0) {
2206                         icl_pdu_free(request);
2207                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2208                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2209                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2210                         }
2211                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2212                         xpt_done(ccb);
2213                         return;
2214                 }
2215         }
2216         iscsi_pdu_queue_locked(request);
2217 }
2218
2219 static void
2220 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2221 {
2222         struct iscsi_session *is;
2223
2224         is = cam_sim_softc(sim);
2225
2226         ISCSI_SESSION_LOCK_ASSERT(is);
2227
2228         if (is->is_terminating ||
2229             (is->is_connected == false && fail_on_disconnection)) {
2230                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2231                 xpt_done(ccb);
2232                 return;
2233         }
2234
2235         switch (ccb->ccb_h.func_code) {
2236         case XPT_PATH_INQ:
2237         {
2238                 struct ccb_pathinq *cpi = &ccb->cpi;
2239
2240                 cpi->version_num = 1;
2241                 cpi->hba_inquiry = PI_TAG_ABLE;
2242                 cpi->target_sprt = 0;
2243                 //cpi->hba_misc = PIM_NOBUSRESET;
2244                 cpi->hba_misc = 0;
2245                 cpi->hba_eng_cnt = 0;
2246                 cpi->max_target = 0;
2247                 cpi->max_lun = 255;
2248                 //cpi->initiator_id = 0; /* XXX */
2249                 cpi->initiator_id = 64; /* XXX */
2250                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2251                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2252                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2253                 cpi->unit_number = cam_sim_unit(sim);
2254                 cpi->bus_id = cam_sim_bus(sim);
2255                 cpi->base_transfer_speed = 150000; /* XXX */
2256                 cpi->transport = XPORT_ISCSI;
2257                 cpi->transport_version = 0;
2258                 cpi->protocol = PROTO_SCSI;
2259                 cpi->protocol_version = SCSI_REV_SPC3;
2260                 cpi->maxio = MAXPHYS;
2261                 cpi->ccb_h.status = CAM_REQ_CMP;
2262                 break;
2263         }
2264         case XPT_GET_TRAN_SETTINGS:
2265         {
2266                 struct ccb_trans_settings       *cts;
2267                 struct ccb_trans_settings_scsi  *scsi;
2268
2269                 cts = &ccb->cts;
2270                 scsi = &cts->proto_specific.scsi;
2271
2272                 cts->protocol = PROTO_SCSI;
2273                 cts->protocol_version = SCSI_REV_SPC3;
2274                 cts->transport = XPORT_ISCSI;
2275                 cts->transport_version = 0;
2276                 scsi->valid = CTS_SCSI_VALID_TQ;
2277                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2278                 cts->ccb_h.status = CAM_REQ_CMP;
2279                 break;
2280         }
2281         case XPT_CALC_GEOMETRY:
2282                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2283                 ccb->ccb_h.status = CAM_REQ_CMP;
2284                 break;
2285 #if 0
2286         /*
2287          * XXX: What's the point?
2288          */
2289         case XPT_RESET_BUS:
2290         case XPT_TERM_IO:
2291                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2292                 ccb->ccb_h.status = CAM_REQ_CMP;
2293                 break;
2294 #endif
2295         case XPT_ABORT:
2296                 iscsi_action_abort(is, ccb);
2297                 return;
2298         case XPT_SCSI_IO:
2299                 iscsi_action_scsiio(is, ccb);
2300                 return;
2301         default:
2302 #if 0
2303                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2304 #endif
2305                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2306                 break;
2307         }
2308         xpt_done(ccb);
2309 }
2310
2311 static void
2312 iscsi_poll(struct cam_sim *sim)
2313 {
2314
2315         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2316 }
2317
2318 static void
2319 iscsi_shutdown(struct iscsi_softc *sc)
2320 {
2321         struct iscsi_session *is;
2322
2323         ISCSI_DEBUG("removing all sessions due to shutdown");
2324
2325         sx_slock(&sc->sc_lock);
2326         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2327                 iscsi_session_terminate(is);
2328         sx_sunlock(&sc->sc_lock);
2329 }
2330
2331 static int
2332 iscsi_load(void)
2333 {
2334         int error;
2335
2336         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2337         sx_init(&sc->sc_lock, "iscsi");
2338         TAILQ_INIT(&sc->sc_sessions);
2339         cv_init(&sc->sc_cv, "iscsi_cv");
2340
2341         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2342             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2343             UMA_ALIGN_PTR, 0);
2344
2345         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2346             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2347         if (error != 0) {
2348                 ISCSI_WARN("failed to create device node, error %d", error);
2349                 return (error);
2350         }
2351         sc->sc_cdev->si_drv1 = sc;
2352
2353         /*
2354          * Note that this needs to get run before dashutdown().  Otherwise,
2355          * when rebooting with iSCSI session with outstanding requests,
2356          * but disconnected, dashutdown() will hang on cam_periph_runccb().
2357          */
2358         sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2359             iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2360
2361         return (0);
2362 }
2363
2364 static int
2365 iscsi_unload(void)
2366 {
2367         struct iscsi_session *is, *tmp;
2368
2369         if (sc->sc_cdev != NULL) {
2370                 ISCSI_DEBUG("removing device node");
2371                 destroy_dev(sc->sc_cdev);
2372                 ISCSI_DEBUG("device node removed");
2373         }
2374
2375         if (sc->sc_shutdown_eh != NULL)
2376                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2377
2378         sx_slock(&sc->sc_lock);
2379         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2380                 iscsi_session_terminate(is);
2381         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2382                 ISCSI_DEBUG("waiting for sessions to terminate");
2383                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2384         }
2385         ISCSI_DEBUG("all sessions terminated");
2386         sx_sunlock(&sc->sc_lock);
2387
2388         uma_zdestroy(iscsi_outstanding_zone);
2389         sx_destroy(&sc->sc_lock);
2390         cv_destroy(&sc->sc_cv);
2391         free(sc, M_ISCSI);
2392         return (0);
2393 }
2394
2395 static int
2396 iscsi_quiesce(void)
2397 {
2398         sx_slock(&sc->sc_lock);
2399         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2400                 sx_sunlock(&sc->sc_lock);
2401                 return (EBUSY);
2402         }
2403         sx_sunlock(&sc->sc_lock);
2404         return (0);
2405 }
2406
2407 static int
2408 iscsi_modevent(module_t mod, int what, void *arg)
2409 {
2410         int error;
2411
2412         switch (what) {
2413         case MOD_LOAD:
2414                 error = iscsi_load();
2415                 break;
2416         case MOD_UNLOAD:
2417                 error = iscsi_unload();
2418                 break;
2419         case MOD_QUIESCE:
2420                 error = iscsi_quiesce();
2421                 break;
2422         default:
2423                 error = EINVAL;
2424                 break;
2425         }
2426         return (error);
2427 }
2428
2429 moduledata_t iscsi_data = {
2430         "iscsi",
2431         iscsi_modevent,
2432         0
2433 };
2434
2435 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2436 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2437 MODULE_DEPEND(iscsi, icl, 1, 1, 1);