sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/condvar.h>
  36 #include <sys/conf.h>
  37 #include <sys/eventhandler.h>
  38 #include <sys/file.h>
  39 #include <sys/kernel.h>
  40 #include <sys/kthread.h>
  41 #include <sys/lock.h>
  42 #include <sys/malloc.h>
  43 #include <sys/mutex.h>
  44 #include <sys/module.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/systm.h>
  47 #include <sys/sx.h>
  48 #include <vm/uma.h>
  49
  50 #include <cam/cam.h>
  51 #include <cam/cam_ccb.h>
  52 #include <cam/cam_xpt.h>
  53 #include <cam/cam_debug.h>
  54 #include <cam/cam_sim.h>
  55 #include <cam/cam_xpt_sim.h>
  56 #include <cam/cam_xpt_periph.h>
  57 #include <cam/cam_periph.h>
  58 #include <cam/scsi/scsi_all.h>
  59 #include <cam/scsi/scsi_message.h>
  60
  61 #include <dev/iscsi/icl.h>
  62 #include <dev/iscsi/iscsi_ioctl.h>
  63 #include <dev/iscsi/iscsi_proto.h>
  64 #include <dev/iscsi/iscsi.h>
  65
  66 #ifdef ICL_KERNEL_PROXY
  67 #include <sys/socketvar.h>
  68 #endif
  69
  70 #ifdef ICL_KERNEL_PROXY
  71 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  72 #endif
  73
  74 /*
  75  * XXX: This is global so the iscsi_unload() can access it.
  76  *      Think about how to do this properly.
  77  */
  78 static struct iscsi_softc       *sc;
  79
  80 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  81 static int debug = 1;
  82 TUNABLE_INT("kern.iscsi.debug", &debug);
  83 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  84     &debug, 0, "Enable debug messages");
  85 static int ping_timeout = 5;
  86 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
  87 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  88     0, "Timeout for ping (NOP-Out) requests, in seconds");
  89 static int iscsid_timeout = 60;
  90 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
  91 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  92     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  93 static int login_timeout = 60;
  94 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
  95 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
  96     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  97 static int maxtags = 255;
  98 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
  99 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
 100     0, "Max number of IO requests queued");
 101 static int fail_on_disconnection = 0;
 102 TUNABLE_INT("kern.iscsi.fail_on_disconnection", &fail_on_disconnection);
 103 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
 104     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
 105
 106 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 107 static uma_zone_t iscsi_outstanding_zone;
 108
 109 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 110 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 111
 112 #define ISCSI_DEBUG(X, ...)                                             \
 113         do {                                                            \
 114                 if (debug > 1)                                          \
 115                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 116         } while (0)
 117
 118 #define ISCSI_WARN(X, ...)                                              \
 119         do {                                                            \
 120                 if (debug > 0) {                                        \
 121                         printf("WARNING: %s: " X "\n",                  \
 122                             __func__, ## __VA_ARGS__);                  \
 123                 }                                                       \
 124         } while (0)
 125
 126 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 127         do {                                                            \
 128                 if (debug > 1) {                                        \
 129                         printf("%s: %s (%s): " X "\n",                  \
 130                             __func__, S->is_conf.isc_target_addr,       \
 131                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 132                 }                                                       \
 133         } while (0)
 134
 135 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 136         do {                                                            \
 137                 if (debug > 0) {                                        \
 138                         printf("WARNING: %s (%s): " X "\n",             \
 139                             S->is_conf.isc_target_addr,                 \
 140                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 141                 }                                                       \
 142         } while (0)
 143
 144 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 145 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 146 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 147 #define ISCSI_SESSION_LOCK_ASSERT_NOT(X) mtx_assert(&X->is_lock, MA_NOTOWNED)
 148
 149 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 150                     int mode, struct thread *td);
 151
 152 static struct cdevsw iscsi_cdevsw = {
 153      .d_version = D_VERSION,
 154      .d_ioctl   = iscsi_ioctl,
 155      .d_name    = "iscsi",
 156 };
 157
 158 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 159 static void     iscsi_pdu_queue(struct icl_pdu *request);
 160 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 161 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 162 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 163 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
 164 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 165 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 166 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 167 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 168 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 169 static void     iscsi_session_reconnect(struct iscsi_session *is);
 170 static void     iscsi_session_terminate(struct iscsi_session *is);
 171 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 172 static void     iscsi_poll(struct cam_sim *sim);
 173 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 174                     uint32_t initiator_task_tag);
 175 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
 176                     uint32_t initiator_task_tag, union ccb *ccb);
 177 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 178                     struct iscsi_outstanding *io);
 179
 180 static bool
 181 iscsi_pdu_prepare(struct icl_pdu *request)
 182 {
 183         struct iscsi_session *is;
 184         struct iscsi_bhs_scsi_command *bhssc;
 185
 186         is = PDU_SESSION(request);
 187
 188         ISCSI_SESSION_LOCK_ASSERT(is);
 189
 190         /*
 191          * We're only using fields common for all the request
 192          * (initiator -> target) PDUs.
 193          */
 194         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 195
 196         /*
 197          * Data-Out PDU does not contain CmdSN.
 198          */
 199         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 200                 if (ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn) &&
 201                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 202                         /*
 203                          * Current MaxCmdSN prevents us from sending any more
 204                          * SCSI Command PDUs to the target; postpone the PDU.
 205                          * It will get resent by either iscsi_pdu_queue(),
 206                          * or by maintenance thread.
 207                          */
 208 #if 0
 209                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %u, "
 210                             "ExpCmdSN %u, MaxCmdSN %u, opcode 0x%x",
 211                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn,
 212                             bhssc->bhssc_opcode);
 213 #endif
 214                         return (true);
 215                 }
 216                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 217                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 218                         is->is_cmdsn++;
 219         }
 220         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 221
 222         return (false);
 223 }
 224
 225 static void
 226 iscsi_session_send_postponed(struct iscsi_session *is)
 227 {
 228         struct icl_pdu *request;
 229         bool postpone;
 230
 231         ISCSI_SESSION_LOCK_ASSERT(is);
 232
 233         while (!STAILQ_EMPTY(&is->is_postponed)) {
 234                 request = STAILQ_FIRST(&is->is_postponed);
 235                 postpone = iscsi_pdu_prepare(request);
 236                 if (postpone)
 237                         break;
 238                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 239                 icl_pdu_queue(request);
 240         }
 241 }
 242
 243 static void
 244 iscsi_pdu_queue_locked(struct icl_pdu *request)
 245 {
 246         struct iscsi_session *is;
 247         bool postpone;
 248
 249         is = PDU_SESSION(request);
 250         ISCSI_SESSION_LOCK_ASSERT(is);
 251         iscsi_session_send_postponed(is);
 252         postpone = iscsi_pdu_prepare(request);
 253         if (postpone) {
 254                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 255                 return;
 256         }
 257         icl_pdu_queue(request);
 258 }
 259
 260 static void
 261 iscsi_pdu_queue(struct icl_pdu *request)
 262 {
 263         struct iscsi_session *is;
 264
 265         is = PDU_SESSION(request);
 266         ISCSI_SESSION_LOCK(is);
 267         iscsi_pdu_queue_locked(request);
 268         ISCSI_SESSION_UNLOCK(is);
 269 }
 270
 271 static void
 272 iscsi_session_logout(struct iscsi_session *is)
 273 {
 274         struct icl_pdu *request;
 275         struct iscsi_bhs_logout_request *bhslr;
 276
 277         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 278         if (request == NULL)
 279                 return;
 280
 281         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 282         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 283         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 284         iscsi_pdu_queue_locked(request);
 285 }
 286
 287 static void
 288 iscsi_session_terminate_task(struct iscsi_session *is,
 289     struct iscsi_outstanding *io, bool requeue)
 290 {
 291
 292         if (io->io_ccb != NULL) {
 293                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
 294                 if (requeue)
 295                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 296                 else
 297                         io->io_ccb->ccb_h.status |= CAM_REQ_ABORTED;
 298                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 299                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 300                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 301                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 302                 }
 303                 xpt_done(io->io_ccb);
 304         }
 305         iscsi_outstanding_remove(is, io);
 306 }
 307
 308 static void
 309 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 310 {
 311         struct iscsi_outstanding *io, *tmp;
 312
 313         ISCSI_SESSION_LOCK_ASSERT(is);
 314
 315         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 316                 iscsi_session_terminate_task(is, io, requeue);
 317         }
 318 }
 319
 320 static void
 321 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 322 {
 323         struct icl_pdu *pdu;
 324
 325         ISCSI_SESSION_LOCK_ASSERT(is);
 326
 327         /*
 328          * Don't queue any new PDUs.
 329          */
 330         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 331                 ISCSI_SESSION_DEBUG(is, "freezing");
 332                 xpt_freeze_simq(is->is_sim, 1);
 333                 is->is_simq_frozen = true;
 334         }
 335
 336         /*
 337          * Remove postponed PDUs.
 338          */
 339         while (!STAILQ_EMPTY(&is->is_postponed)) {
 340                 pdu = STAILQ_FIRST(&is->is_postponed);
 341                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 342                 icl_pdu_free(pdu);
 343         }
 344
 345         if (destroy_sim == false) {
 346                 /*
 347                  * Terminate SCSI tasks, asking CAM to requeue them.
 348                  */
 349                 iscsi_session_terminate_tasks(is, true);
 350                 return;
 351         }
 352
 353         iscsi_session_terminate_tasks(is, false);
 354
 355         if (is->is_sim == NULL)
 356                 return;
 357
 358         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 359         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 360
 361         if (is->is_simq_frozen) {
 362                 xpt_release_simq(is->is_sim, 1);
 363                 is->is_simq_frozen = false;
 364         }
 365
 366         xpt_free_path(is->is_path);
 367         is->is_path = NULL;
 368         xpt_bus_deregister(cam_sim_path(is->is_sim));
 369         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 370         is->is_sim = NULL;
 371         is->is_devq = NULL;
 372 }
 373
 374 static void
 375 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 376 {
 377
 378         icl_conn_close(is->is_conn);
 379
 380         ISCSI_SESSION_LOCK(is);
 381
 382         is->is_connected = false;
 383         is->is_reconnecting = false;
 384         is->is_login_phase = false;
 385
 386 #ifdef ICL_KERNEL_PROXY
 387         if (is->is_login_pdu != NULL) {
 388                 icl_pdu_free(is->is_login_pdu);
 389                 is->is_login_pdu = NULL;
 390         }
 391         cv_signal(&is->is_login_cv);
 392 #endif
 393
 394         if (fail_on_disconnection) {
 395                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
 396                 iscsi_session_cleanup(is, true);
 397         } else {
 398                 iscsi_session_cleanup(is, false);
 399         }
 400
 401         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 402             ("destroying session with active tasks"));
 403         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 404             ("destroying session with postponed PDUs"));
 405
 406         /*
 407          * Request immediate reconnection from iscsid(8).
 408          */
 409         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 410         is->is_waiting_for_iscsid = true;
 411         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 412         is->is_timeout = 0;
 413         ISCSI_SESSION_UNLOCK(is);
 414         cv_signal(&is->is_softc->sc_cv);
 415 }
 416
 417 static void
 418 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 419 {
 420         struct iscsi_softc *sc;
 421
 422         sc = is->is_softc;
 423         sx_xlock(&sc->sc_lock);
 424         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 425         sx_xunlock(&sc->sc_lock);
 426
 427         icl_conn_close(is->is_conn);
 428         callout_drain(&is->is_callout);
 429
 430         ISCSI_SESSION_LOCK(is);
 431
 432         KASSERT(is->is_terminating, ("is_terminating == false"));
 433
 434 #ifdef ICL_KERNEL_PROXY
 435         if (is->is_login_pdu != NULL) {
 436                 icl_pdu_free(is->is_login_pdu);
 437                 is->is_login_pdu = NULL;
 438         }
 439         cv_signal(&is->is_login_cv);
 440 #endif
 441
 442         iscsi_session_cleanup(is, true);
 443
 444         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 445             ("destroying session with active tasks"));
 446         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 447             ("destroying session with postponed PDUs"));
 448
 449         ISCSI_SESSION_UNLOCK(is);
 450
 451         icl_conn_free(is->is_conn);
 452         mtx_destroy(&is->is_lock);
 453         cv_destroy(&is->is_maintenance_cv);
 454 #ifdef ICL_KERNEL_PROXY
 455         cv_destroy(&is->is_login_cv);
 456 #endif
 457         ISCSI_SESSION_DEBUG(is, "terminated");
 458         free(is, M_ISCSI);
 459
 460         /*
 461          * The iscsi_unload() routine might be waiting.
 462          */
 463         cv_signal(&sc->sc_cv);
 464 }
 465
 466 static void
 467 iscsi_maintenance_thread(void *arg)
 468 {
 469         struct iscsi_session *is;
 470
 471         is = arg;
 472
 473         for (;;) {
 474                 ISCSI_SESSION_LOCK(is);
 475                 if (is->is_reconnecting == false &&
 476                     is->is_terminating == false &&
 477                     STAILQ_EMPTY(&is->is_postponed))
 478                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 479
 480                 if (is->is_reconnecting) {
 481                         ISCSI_SESSION_UNLOCK(is);
 482                         iscsi_maintenance_thread_reconnect(is);
 483                         continue;
 484                 }
 485
 486                 if (is->is_terminating) {
 487                         ISCSI_SESSION_UNLOCK(is);
 488                         iscsi_maintenance_thread_terminate(is);
 489                         kthread_exit();
 490                         return;
 491                 }
 492
 493                 iscsi_session_send_postponed(is);
 494                 ISCSI_SESSION_UNLOCK(is);
 495         }
 496 }
 497
 498 static void
 499 iscsi_session_reconnect(struct iscsi_session *is)
 500 {
 501
 502         /*
 503          * XXX: We can't use locking here, because
 504          *      it's being called from various contexts.
 505          *      Hope it doesn't break anything.
 506          */
 507         if (is->is_reconnecting)
 508                 return;
 509
 510         is->is_reconnecting = true;
 511         cv_signal(&is->is_maintenance_cv);
 512 }
 513
 514 static void
 515 iscsi_session_terminate(struct iscsi_session *is)
 516 {
 517
 518         if (is->is_terminating)
 519                 return;
 520
 521         is->is_terminating = true;
 522
 523 #if 0
 524         iscsi_session_logout(is);
 525 #endif
 526         cv_signal(&is->is_maintenance_cv);
 527 }
 528
 529 static void
 530 iscsi_callout(void *context)
 531 {
 532         struct icl_pdu *request;
 533         struct iscsi_bhs_nop_out *bhsno;
 534         struct iscsi_session *is;
 535         bool reconnect_needed = false;
 536
 537         is = context;
 538
 539         ISCSI_SESSION_LOCK(is);
 540         if (is->is_terminating) {
 541                 ISCSI_SESSION_UNLOCK(is);
 542                 return;
 543         }
 544
 545         callout_schedule(&is->is_callout, 1 * hz);
 546
 547         is->is_timeout++;
 548
 549         if (is->is_waiting_for_iscsid) {
 550                 if (iscsid_timeout > 0 && is->is_timeout > iscsid_timeout) {
 551                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 552                             "for %d seconds; reconnecting",
 553                             is->is_timeout);
 554                         reconnect_needed = true;
 555                 }
 556                 goto out;
 557         }
 558
 559         if (is->is_login_phase) {
 560                 if (login_timeout > 0 && is->is_timeout > login_timeout) {
 561                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 562                             "reconnecting", is->is_timeout);
 563                         reconnect_needed = true;
 564                 }
 565                 goto out;
 566         }
 567
 568         if (ping_timeout <= 0) {
 569                 /*
 570                  * Pings are disabled.  Don't send NOP-Out in this case.
 571                  * Reset the timeout, to avoid triggering reconnection,
 572                  * should the user decide to reenable them.
 573                  */
 574                 is->is_timeout = 0;
 575                 goto out;
 576         }
 577
 578         if (is->is_timeout >= ping_timeout) {
 579                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 580                     "reconnecting", ping_timeout);
 581                 reconnect_needed = true;
 582                 goto out;
 583         }
 584
 585         ISCSI_SESSION_UNLOCK(is);
 586
 587         /*
 588          * If the ping was reset less than one second ago - which means
 589          * that we've received some PDU during the last second - assume
 590          * the traffic flows correctly and don't bother sending a NOP-Out.
 591          *
 592          * (It's 2 - one for one second, and one for incrementing is_timeout
 593          * earlier in this routine.)
 594          */
 595         if (is->is_timeout < 2)
 596                 return;
 597
 598         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 599         if (request == NULL) {
 600                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 601                 return;
 602         }
 603         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 604         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 605             ISCSI_BHS_OPCODE_IMMEDIATE;
 606         bhsno->bhsno_flags = 0x80;
 607         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 608         iscsi_pdu_queue(request);
 609         return;
 610
 611 out:
 612         ISCSI_SESSION_UNLOCK(is);
 613
 614         if (reconnect_needed)
 615                 iscsi_session_reconnect(is);
 616 }
 617
 618 static void
 619 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 620 {
 621         const struct iscsi_bhs_data_in *bhsdi;
 622         struct iscsi_session *is;
 623         uint32_t expcmdsn, maxcmdsn, statsn;
 624
 625         is = PDU_SESSION(response);
 626
 627         ISCSI_SESSION_LOCK_ASSERT(is);
 628
 629         /*
 630          * We're only using fields common for all the response
 631          * (target -> initiator) PDUs.
 632          */
 633         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 634         /*
 635          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 636          * and Residual Count only have meaningful content if the S bit
 637          * is set to 1", so we also need to check the bit specific for
 638          * Data-In PDU.
 639          */
 640         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 641             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 642                 statsn = ntohl(bhsdi->bhsdi_statsn);
 643                 if (statsn != is->is_statsn && statsn != (is->is_statsn + 1)) {
 644                         /* XXX: This is normal situation for MCS */
 645                         ISCSI_SESSION_WARN(is, "PDU 0x%x StatSN %u != "
 646                             "session ExpStatSN %u (or + 1); reconnecting",
 647                             bhsdi->bhsdi_opcode, statsn, is->is_statsn);
 648                         iscsi_session_reconnect(is);
 649                 }
 650                 if (ISCSI_SNGT(statsn, is->is_statsn))
 651                         is->is_statsn = statsn;
 652         }
 653
 654         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 655         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 656
 657         if (ISCSI_SNLT(maxcmdsn + 1, expcmdsn)) {
 658                 ISCSI_SESSION_DEBUG(is,
 659                     "PDU MaxCmdSN %u + 1 < PDU ExpCmdSN %u; ignoring",
 660                     maxcmdsn, expcmdsn);
 661         } else {
 662                 if (ISCSI_SNGT(maxcmdsn, is->is_maxcmdsn)) {
 663                         is->is_maxcmdsn = maxcmdsn;
 664
 665                         /*
 666                          * Command window increased; kick the maintanance thread
 667                          * to send out postponed commands.
 668                          */
 669                         if (!STAILQ_EMPTY(&is->is_postponed))
 670                                 cv_signal(&is->is_maintenance_cv);
 671                 } else if (ISCSI_SNLT(maxcmdsn, is->is_maxcmdsn)) {
 672                         /* XXX: This is normal situation for MCS */
 673                         ISCSI_SESSION_DEBUG(is,
 674                             "PDU MaxCmdSN %u < session MaxCmdSN %u; ignoring",
 675                             maxcmdsn, is->is_maxcmdsn);
 676                 }
 677
 678                 if (ISCSI_SNGT(expcmdsn, is->is_expcmdsn)) {
 679                         is->is_expcmdsn = expcmdsn;
 680                 } else if (ISCSI_SNLT(expcmdsn, is->is_expcmdsn)) {
 681                         /* XXX: This is normal situation for MCS */
 682                         ISCSI_SESSION_DEBUG(is,
 683                             "PDU ExpCmdSN %u < session ExpCmdSN %u; ignoring",
 684                             expcmdsn, is->is_expcmdsn);
 685                 }
 686         }
 687
 688         /*
 689          * Every incoming PDU - not just NOP-In - resets the ping timer.
 690          * The purpose of the timeout is to reset the connection when it stalls;
 691          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 692          * in some queue.
 693          */
 694         is->is_timeout = 0;
 695 }
 696
 697 static void
 698 iscsi_receive_callback(struct icl_pdu *response)
 699 {
 700         struct iscsi_session *is;
 701
 702         is = PDU_SESSION(response);
 703
 704         ISCSI_SESSION_LOCK(is);
 705
 706 #ifdef ICL_KERNEL_PROXY
 707         if (is->is_login_phase) {
 708                 if (is->is_login_pdu == NULL)
 709                         is->is_login_pdu = response;
 710                 else
 711                         icl_pdu_free(response);
 712                 ISCSI_SESSION_UNLOCK(is);
 713                 cv_signal(&is->is_login_cv);
 714                 return;
 715         }
 716 #endif
 717
 718         iscsi_pdu_update_statsn(response);
 719
 720         /*
 721          * The handling routine is responsible for freeing the PDU
 722          * when it's no longer needed.
 723          */
 724         switch (response->ip_bhs->bhs_opcode) {
 725         case ISCSI_BHS_OPCODE_NOP_IN:
 726                 iscsi_pdu_handle_nop_in(response);
 727                 ISCSI_SESSION_UNLOCK(is);
 728                 break;
 729         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 730                 iscsi_pdu_handle_scsi_response(response);
 731                 /* Session lock dropped inside. */
 732                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 733                 break;
 734         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
 735                 iscsi_pdu_handle_task_response(response);
 736                 ISCSI_SESSION_UNLOCK(is);
 737                 break;
 738         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 739                 iscsi_pdu_handle_data_in(response);
 740                 /* Session lock dropped inside. */
 741                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 742                 break;
 743         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 744                 iscsi_pdu_handle_logout_response(response);
 745                 ISCSI_SESSION_UNLOCK(is);
 746                 break;
 747         case ISCSI_BHS_OPCODE_R2T:
 748                 iscsi_pdu_handle_r2t(response);
 749                 ISCSI_SESSION_UNLOCK(is);
 750                 break;
 751         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 752                 iscsi_pdu_handle_async_message(response);
 753                 ISCSI_SESSION_UNLOCK(is);
 754                 break;
 755         case ISCSI_BHS_OPCODE_REJECT:
 756                 iscsi_pdu_handle_reject(response);
 757                 ISCSI_SESSION_UNLOCK(is);
 758                 break;
 759         default:
 760                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 761                     "opcode 0x%x; reconnecting",
 762                     response->ip_bhs->bhs_opcode);
 763                 iscsi_session_reconnect(is);
 764                 ISCSI_SESSION_UNLOCK(is);
 765                 icl_pdu_free(response);
 766         }
 767 }
 768
 769 static void
 770 iscsi_error_callback(struct icl_conn *ic)
 771 {
 772         struct iscsi_session *is;
 773
 774         is = CONN_SESSION(ic);
 775
 776         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 777         iscsi_session_reconnect(is);
 778 }
 779
 780 static void
 781 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 782 {
 783         struct iscsi_session *is;
 784         struct iscsi_bhs_nop_out *bhsno;
 785         struct iscsi_bhs_nop_in *bhsni;
 786         struct icl_pdu *request;
 787         void *data = NULL;
 788         size_t datasize;
 789         int error;
 790
 791         is = PDU_SESSION(response);
 792         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 793
 794         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 795                 /*
 796                  * Nothing to do; iscsi_pdu_update_statsn() already
 797                  * zeroed the timeout.
 798                  */
 799                 icl_pdu_free(response);
 800                 return;
 801         }
 802
 803         datasize = icl_pdu_data_segment_length(response);
 804         if (datasize > 0) {
 805                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 806                 if (data == NULL) {
 807                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 808                             "reconnecting");
 809                         icl_pdu_free(response);
 810                         iscsi_session_reconnect(is);
 811                         return;
 812                 }
 813                 icl_pdu_get_data(response, 0, data, datasize);
 814         }
 815
 816         request = icl_pdu_new(response->ip_conn, M_NOWAIT);
 817         if (request == NULL) {
 818                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 819                     "reconnecting");
 820                 free(data, M_ISCSI);
 821                 icl_pdu_free(response);
 822                 iscsi_session_reconnect(is);
 823                 return;
 824         }
 825         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 826         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 827             ISCSI_BHS_OPCODE_IMMEDIATE;
 828         bhsno->bhsno_flags = 0x80;
 829         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 830         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 831         if (datasize > 0) {
 832                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 833                 if (error != 0) {
 834                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 835                             "reconnecting");
 836                         free(data, M_ISCSI);
 837                         icl_pdu_free(request);
 838                         icl_pdu_free(response);
 839                         iscsi_session_reconnect(is);
 840                         return;
 841                 }
 842                 free(data, M_ISCSI);
 843         }
 844
 845         icl_pdu_free(response);
 846         iscsi_pdu_queue_locked(request);
 847 }
 848
 849 static void
 850 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 851 {
 852         struct iscsi_bhs_scsi_response *bhssr;
 853         struct iscsi_outstanding *io;
 854         struct iscsi_session *is;
 855         union ccb *ccb;
 856         struct ccb_scsiio *csio;
 857         size_t data_segment_len, received;
 858         uint16_t sense_len;
 859
 860         is = PDU_SESSION(response);
 861
 862         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 863         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 864         if (io == NULL || io->io_ccb == NULL) {
 865                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 866                 icl_pdu_free(response);
 867                 iscsi_session_reconnect(is);
 868                 ISCSI_SESSION_UNLOCK(is);
 869                 return;
 870         }
 871
 872         ccb = io->io_ccb;
 873         received = io->io_received;
 874         iscsi_outstanding_remove(is, io);
 875         ISCSI_SESSION_UNLOCK(is);
 876
 877         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 878                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 879                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 880                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 881                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 882                 }
 883                 ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 884         } else if (bhssr->bhssr_status == 0) {
 885                 ccb->ccb_h.status = CAM_REQ_CMP;
 886         } else {
 887                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 888                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 889                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 890                 }
 891                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 892                 ccb->csio.scsi_status = bhssr->bhssr_status;
 893         }
 894
 895         csio = &ccb->csio;
 896         data_segment_len = icl_pdu_data_segment_length(response);
 897         if (data_segment_len > 0) {
 898                 if (data_segment_len < sizeof(sense_len)) {
 899                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 900                             data_segment_len);
 901                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 902                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 903                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 904                         }
 905                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 906                         goto out;
 907                 }
 908                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 909                 sense_len = ntohs(sense_len);
 910 #if 0
 911                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 912                     sense_len, data_segment_len);
 913 #endif
 914                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 915                         ISCSI_SESSION_WARN(is, "truncated data segment "
 916                             "(%zd bytes, should be %zd)",
 917                             data_segment_len, sizeof(sense_len) + sense_len);
 918                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 919                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 920                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 921                         }
 922                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 923                         goto out;
 924                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 925                         ISCSI_SESSION_WARN(is, "oversize data segment "
 926                             "(%zd bytes, should be %zd)",
 927                             data_segment_len, sizeof(sense_len) + sense_len);
 928                 if (sense_len > csio->sense_len) {
 929                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 930                             sense_len, csio->sense_len);
 931                         sense_len = csio->sense_len;
 932                 }
 933                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 934                 csio->sense_resid = csio->sense_len - sense_len;
 935                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 936         }
 937
 938 out:
 939         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 940                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 941
 942         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 943                 KASSERT(received <= csio->dxfer_len,
 944                     ("received > csio->dxfer_len"));
 945                 if (received < csio->dxfer_len) {
 946                         if (csio->resid != csio->dxfer_len - received) {
 947                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 948                                     "target indicates %d, we calculated %zd",
 949                                     csio->resid, csio->dxfer_len - received);
 950                         }
 951                         csio->resid = csio->dxfer_len - received;
 952                 }
 953         }
 954
 955         xpt_done(ccb);
 956         icl_pdu_free(response);
 957 }
 958
 959 static void
 960 iscsi_pdu_handle_task_response(struct icl_pdu *response)
 961 {
 962         struct iscsi_bhs_task_management_response *bhstmr;
 963         struct iscsi_outstanding *io, *aio;
 964         struct iscsi_session *is;
 965
 966         is = PDU_SESSION(response);
 967
 968         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
 969         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
 970         if (io == NULL || io->io_ccb != NULL) {
 971                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
 972                     bhstmr->bhstmr_initiator_task_tag);
 973                 icl_pdu_free(response);
 974                 iscsi_session_reconnect(is);
 975                 return;
 976         }
 977
 978         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
 979                 ISCSI_SESSION_WARN(is, "task response 0x%x",
 980                     bhstmr->bhstmr_response);
 981         } else {
 982                 aio = iscsi_outstanding_find(is, io->io_datasn);
 983                 if (aio != NULL && aio->io_ccb != NULL)
 984                         iscsi_session_terminate_task(is, aio, false);
 985         }
 986
 987         iscsi_outstanding_remove(is, io);
 988         icl_pdu_free(response);
 989 }
 990
 991 static void
 992 iscsi_pdu_handle_data_in(struct icl_pdu *response)
 993 {
 994         struct iscsi_bhs_data_in *bhsdi;
 995         struct iscsi_outstanding *io;
 996         struct iscsi_session *is;
 997         union ccb *ccb;
 998         struct ccb_scsiio *csio;
 999         size_t data_segment_len, received, oreceived;
1000
1001         is = PDU_SESSION(response);
1002         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
1003         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
1004         if (io == NULL || io->io_ccb == NULL) {
1005                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
1006                 icl_pdu_free(response);
1007                 iscsi_session_reconnect(is);
1008                 ISCSI_SESSION_UNLOCK(is);
1009                 return;
1010         }
1011
1012         data_segment_len = icl_pdu_data_segment_length(response);
1013         if (data_segment_len == 0) {
1014                 /*
1015                  * "The sending of 0 length data segments should be avoided,
1016                  * but initiators and targets MUST be able to properly receive
1017                  * 0 length data segments."
1018                  */
1019                 ISCSI_SESSION_UNLOCK(is);
1020                 icl_pdu_free(response);
1021                 return;
1022         }
1023
1024         /*
1025          * We need to track this for security reasons - without it, malicious target
1026          * could respond to SCSI READ without sending Data-In PDUs, which would result
1027          * in read operation on the initiator side returning random kernel data.
1028          */
1029         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1030                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1031                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1032                 icl_pdu_free(response);
1033                 iscsi_session_reconnect(is);
1034                 ISCSI_SESSION_UNLOCK(is);
1035                 return;
1036         }
1037
1038         ccb = io->io_ccb;
1039         csio = &ccb->csio;
1040
1041         if (io->io_received + data_segment_len > csio->dxfer_len) {
1042                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1043                     "at offset %zd, buffer is %d)",
1044                     data_segment_len, io->io_received, csio->dxfer_len);
1045                 icl_pdu_free(response);
1046                 iscsi_session_reconnect(is);
1047                 ISCSI_SESSION_UNLOCK(is);
1048                 return;
1049         }
1050
1051         oreceived = io->io_received;
1052         io->io_received += data_segment_len;
1053         received = io->io_received;
1054         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0)
1055                 iscsi_outstanding_remove(is, io);
1056         ISCSI_SESSION_UNLOCK(is);
1057
1058         icl_pdu_get_data(response, 0, csio->data_ptr + oreceived, data_segment_len);
1059
1060         /*
1061          * XXX: Check DataSN.
1062          * XXX: Check F.
1063          */
1064         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1065                 /*
1066                  * Nothing more to do.
1067                  */
1068                 icl_pdu_free(response);
1069                 return;
1070         }
1071
1072         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1073         if (bhsdi->bhsdi_status == 0) {
1074                 ccb->ccb_h.status = CAM_REQ_CMP;
1075         } else {
1076                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1077                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1078                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1079                 }
1080                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1081                 csio->scsi_status = bhsdi->bhsdi_status;
1082         }
1083
1084         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1085                 KASSERT(received <= csio->dxfer_len,
1086                     ("received > csio->dxfer_len"));
1087                 if (received < csio->dxfer_len) {
1088                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1089                         if (csio->resid != csio->dxfer_len - received) {
1090                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1091                                     "target indicates %d, we calculated %zd",
1092                                     csio->resid, csio->dxfer_len - received);
1093                         }
1094                         csio->resid = csio->dxfer_len - received;
1095                 }
1096         }
1097
1098         xpt_done(ccb);
1099         icl_pdu_free(response);
1100 }
1101
1102 static void
1103 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1104 {
1105
1106         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1107         icl_pdu_free(response);
1108 }
1109
1110 static void
1111 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1112 {
1113         struct icl_pdu *request;
1114         struct iscsi_session *is;
1115         struct iscsi_bhs_r2t *bhsr2t;
1116         struct iscsi_bhs_data_out *bhsdo;
1117         struct iscsi_outstanding *io;
1118         struct ccb_scsiio *csio;
1119         size_t off, len, total_len;
1120         int error;
1121
1122         is = PDU_SESSION(response);
1123
1124         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1125         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1126         if (io == NULL || io->io_ccb == NULL) {
1127                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1128                     bhsr2t->bhsr2t_initiator_task_tag);
1129                 icl_pdu_free(response);
1130                 iscsi_session_reconnect(is);
1131                 return;
1132         }
1133
1134         csio = &io->io_ccb->csio;
1135
1136         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1137                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1138                 icl_pdu_free(response);
1139                 iscsi_session_reconnect(is);
1140                 return;
1141         }
1142
1143         /*
1144          * XXX: Verify R2TSN.
1145          */
1146
1147         io->io_datasn = 0;
1148
1149         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1150         if (off > csio->dxfer_len) {
1151                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1152                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1153                 icl_pdu_free(response);
1154                 iscsi_session_reconnect(is);
1155                 return;
1156         }
1157
1158         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1159         if (total_len == 0 || total_len > csio->dxfer_len) {
1160                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1161                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1162                 icl_pdu_free(response);
1163                 iscsi_session_reconnect(is);
1164                 return;
1165         }
1166
1167         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1168
1169         for (;;) {
1170                 len = total_len;
1171
1172                 if (len > is->is_max_data_segment_length)
1173                         len = is->is_max_data_segment_length;
1174
1175                 if (off + len > csio->dxfer_len) {
1176                         ISCSI_SESSION_WARN(is, "target requested invalid "
1177                             "length/offset %zd, buffer is %d; reconnecting",
1178                             off + len, csio->dxfer_len);
1179                         icl_pdu_free(response);
1180                         iscsi_session_reconnect(is);
1181                         return;
1182                 }
1183
1184                 request = icl_pdu_new(response->ip_conn, M_NOWAIT);
1185                 if (request == NULL) {
1186                         icl_pdu_free(response);
1187                         iscsi_session_reconnect(is);
1188                         return;
1189                 }
1190
1191                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1192                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1193                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1194                 bhsdo->bhsdo_initiator_task_tag =
1195                     bhsr2t->bhsr2t_initiator_task_tag;
1196                 bhsdo->bhsdo_target_transfer_tag =
1197                     bhsr2t->bhsr2t_target_transfer_tag;
1198                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1199                 bhsdo->bhsdo_buffer_offset = htonl(off);
1200                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1201                     M_NOWAIT);
1202                 if (error != 0) {
1203                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1204                             "reconnecting");
1205                         icl_pdu_free(request);
1206                         icl_pdu_free(response);
1207                         iscsi_session_reconnect(is);
1208                         return;
1209                 }
1210
1211                 off += len;
1212                 total_len -= len;
1213
1214                 if (total_len == 0) {
1215                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1216                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1217                 } else {
1218                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1219                 }
1220
1221                 iscsi_pdu_queue_locked(request);
1222
1223                 if (total_len == 0)
1224                         break;
1225         }
1226
1227         icl_pdu_free(response);
1228 }
1229
1230 static void
1231 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1232 {
1233         struct iscsi_bhs_asynchronous_message *bhsam;
1234         struct iscsi_session *is;
1235
1236         is = PDU_SESSION(response);
1237         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1238         switch (bhsam->bhsam_async_event) {
1239         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1240                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1241                 iscsi_session_logout(is);
1242                 iscsi_session_terminate(is);
1243                 break;
1244         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1245                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1246                 break;
1247         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1248                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1249                 break;
1250         default:
1251                 /*
1252                  * XXX: Technically, we're obligated to also handle
1253                  *      parameter renegotiation.
1254                  */
1255                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1256                 break;
1257         }
1258
1259         icl_pdu_free(response);
1260 }
1261
1262 static void
1263 iscsi_pdu_handle_reject(struct icl_pdu *response)
1264 {
1265         struct iscsi_bhs_reject *bhsr;
1266         struct iscsi_session *is;
1267
1268         is = PDU_SESSION(response);
1269         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1270         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1271             bhsr->bhsr_reason);
1272
1273         icl_pdu_free(response);
1274 }
1275
1276 static int
1277 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1278     struct iscsi_daemon_request *request)
1279 {
1280         struct iscsi_session *is;
1281         int error;
1282
1283         sx_slock(&sc->sc_lock);
1284         for (;;) {
1285                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1286                         ISCSI_SESSION_LOCK(is);
1287                         if (is->is_waiting_for_iscsid)
1288                                 break;
1289                         ISCSI_SESSION_UNLOCK(is);
1290                 }
1291
1292                 if (is == NULL) {
1293                         /*
1294                          * No session requires attention from iscsid(8); wait.
1295                          */
1296                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1297                         if (error != 0) {
1298                                 sx_sunlock(&sc->sc_lock);
1299                                 return (error);
1300                         }
1301                         continue;
1302                 }
1303
1304                 is->is_waiting_for_iscsid = false;
1305                 is->is_login_phase = true;
1306                 is->is_reason[0] = '\0';
1307                 ISCSI_SESSION_UNLOCK(is);
1308
1309                 request->idr_session_id = is->is_id;
1310                 memcpy(&request->idr_isid, &is->is_isid,
1311                     sizeof(request->idr_isid));
1312                 request->idr_tsih = 0;  /* New or reinstated session. */
1313                 memcpy(&request->idr_conf, &is->is_conf,
1314                     sizeof(request->idr_conf));
1315
1316                 sx_sunlock(&sc->sc_lock);
1317                 return (0);
1318         }
1319 }
1320
1321 static int
1322 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1323     struct iscsi_daemon_handoff *handoff)
1324 {
1325         struct iscsi_session *is;
1326         int error;
1327
1328         sx_slock(&sc->sc_lock);
1329
1330         /*
1331          * Find the session to hand off socket to.
1332          */
1333         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1334                 if (is->is_id == handoff->idh_session_id)
1335                         break;
1336         }
1337         if (is == NULL) {
1338                 sx_sunlock(&sc->sc_lock);
1339                 return (ESRCH);
1340         }
1341         ISCSI_SESSION_LOCK(is);
1342         if (is->is_conf.isc_discovery || is->is_terminating) {
1343                 ISCSI_SESSION_UNLOCK(is);
1344                 sx_sunlock(&sc->sc_lock);
1345                 return (EINVAL);
1346         }
1347         if (is->is_connected) {
1348                 /*
1349                  * This might have happened because another iscsid(8)
1350                  * instance handed off the connection in the meantime.
1351                  * Just return.
1352                  */
1353                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1354                     "session");
1355                 ISCSI_SESSION_UNLOCK(is);
1356                 sx_sunlock(&sc->sc_lock);
1357                 return (EBUSY);
1358         }
1359
1360         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1361             sizeof(is->is_target_alias));
1362         is->is_tsih = handoff->idh_tsih;
1363         is->is_statsn = handoff->idh_statsn;
1364         is->is_initial_r2t = handoff->idh_initial_r2t;
1365         is->is_immediate_data = handoff->idh_immediate_data;
1366         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1367         is->is_max_burst_length = handoff->idh_max_burst_length;
1368         is->is_first_burst_length = handoff->idh_first_burst_length;
1369
1370         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1371                 is->is_conn->ic_header_crc32c = true;
1372         else
1373                 is->is_conn->ic_header_crc32c = false;
1374         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1375                 is->is_conn->ic_data_crc32c = true;
1376         else
1377                 is->is_conn->ic_data_crc32c = false;
1378
1379         is->is_cmdsn = 0;
1380         is->is_expcmdsn = 0;
1381         is->is_maxcmdsn = 0;
1382         is->is_waiting_for_iscsid = false;
1383         is->is_login_phase = false;
1384         is->is_timeout = 0;
1385         is->is_connected = true;
1386         is->is_reason[0] = '\0';
1387
1388         ISCSI_SESSION_UNLOCK(is);
1389
1390 #ifdef ICL_KERNEL_PROXY
1391         if (handoff->idh_socket != 0) {
1392 #endif
1393                 /*
1394                  * Handoff without using ICL proxy.
1395                  */
1396                 error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1397                 if (error != 0) {
1398                         sx_sunlock(&sc->sc_lock);
1399                         iscsi_session_terminate(is);
1400                         return (error);
1401                 }
1402 #ifdef ICL_KERNEL_PROXY
1403         }
1404 #endif
1405
1406         sx_sunlock(&sc->sc_lock);
1407
1408         if (is->is_sim != NULL) {
1409                 /*
1410                  * When reconnecting, there already is SIM allocated for the session.
1411                  */
1412                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1413                 ISCSI_SESSION_LOCK(is);
1414                 ISCSI_SESSION_DEBUG(is, "releasing");
1415                 xpt_release_simq(is->is_sim, 1);
1416                 is->is_simq_frozen = false;
1417                 ISCSI_SESSION_UNLOCK(is);
1418
1419         } else {
1420                 ISCSI_SESSION_LOCK(is);
1421                 is->is_devq = cam_simq_alloc(maxtags);
1422                 if (is->is_devq == NULL) {
1423                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1424                         iscsi_session_terminate(is);
1425                         return (ENOMEM);
1426                 }
1427
1428                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1429                     is, is->is_id /* unit */, &is->is_lock,
1430                     1, maxtags, is->is_devq);
1431                 if (is->is_sim == NULL) {
1432                         ISCSI_SESSION_UNLOCK(is);
1433                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1434                         cam_simq_free(is->is_devq);
1435                         iscsi_session_terminate(is);
1436                         return (ENOMEM);
1437                 }
1438
1439                 error = xpt_bus_register(is->is_sim, NULL, 0);
1440                 if (error != 0) {
1441                         ISCSI_SESSION_UNLOCK(is);
1442                         ISCSI_SESSION_WARN(is, "failed to register bus");
1443                         iscsi_session_terminate(is);
1444                         return (ENOMEM);
1445                 }
1446
1447                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1448                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1449                     CAM_LUN_WILDCARD);
1450                 if (error != CAM_REQ_CMP) {
1451                         ISCSI_SESSION_UNLOCK(is);
1452                         ISCSI_SESSION_WARN(is, "failed to create path");
1453                         iscsi_session_terminate(is);
1454                         return (ENOMEM);
1455                 }
1456                 ISCSI_SESSION_UNLOCK(is);
1457         }
1458
1459         return (0);
1460 }
1461
1462 static int
1463 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1464     struct iscsi_daemon_fail *fail)
1465 {
1466         struct iscsi_session *is;
1467
1468         sx_slock(&sc->sc_lock);
1469
1470         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1471                 if (is->is_id == fail->idf_session_id)
1472                         break;
1473         }
1474         if (is == NULL) {
1475                 sx_sunlock(&sc->sc_lock);
1476                 return (ESRCH);
1477         }
1478         ISCSI_SESSION_LOCK(is);
1479         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1480             fail->idf_reason);
1481         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1482         //is->is_waiting_for_iscsid = false;
1483         //is->is_login_phase = true;
1484         //iscsi_session_reconnect(is);
1485         ISCSI_SESSION_UNLOCK(is);
1486         sx_sunlock(&sc->sc_lock);
1487
1488         return (0);
1489 }
1490
1491 #ifdef ICL_KERNEL_PROXY
1492 static int
1493 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1494     struct iscsi_daemon_connect *idc)
1495 {
1496         struct iscsi_session *is;
1497         struct sockaddr *from_sa, *to_sa;
1498         int error;
1499
1500         sx_slock(&sc->sc_lock);
1501         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1502                 if (is->is_id == idc->idc_session_id)
1503                         break;
1504         }
1505         if (is == NULL) {
1506                 sx_sunlock(&sc->sc_lock);
1507                 return (ESRCH);
1508         }
1509         sx_sunlock(&sc->sc_lock);
1510
1511         if (idc->idc_from_addrlen > 0) {
1512                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1513                 if (error != 0) {
1514                         ISCSI_SESSION_WARN(is,
1515                             "getsockaddr failed with error %d", error);
1516                         return (error);
1517                 }
1518         } else {
1519                 from_sa = NULL;
1520         }
1521         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1522         if (error != 0) {
1523                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1524                     error);
1525                 free(from_sa, M_SONAME);
1526                 return (error);
1527         }
1528
1529         ISCSI_SESSION_LOCK(is);
1530         is->is_waiting_for_iscsid = false;
1531         is->is_login_phase = true;
1532         is->is_timeout = 0;
1533         ISCSI_SESSION_UNLOCK(is);
1534
1535         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1536             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1537         free(from_sa, M_SONAME);
1538         free(to_sa, M_SONAME);
1539
1540         /*
1541          * Digests are always disabled during login phase.
1542          */
1543         is->is_conn->ic_header_crc32c = false;
1544         is->is_conn->ic_data_crc32c = false;
1545
1546         return (error);
1547 }
1548
1549 static int
1550 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1551     struct iscsi_daemon_send *ids)
1552 {
1553         struct iscsi_session *is;
1554         struct icl_pdu *ip;
1555         size_t datalen;
1556         void *data;
1557         int error;
1558
1559         sx_slock(&sc->sc_lock);
1560         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1561                 if (is->is_id == ids->ids_session_id)
1562                         break;
1563         }
1564         if (is == NULL) {
1565                 sx_sunlock(&sc->sc_lock);
1566                 return (ESRCH);
1567         }
1568         sx_sunlock(&sc->sc_lock);
1569
1570         if (is->is_login_phase == false)
1571                 return (EBUSY);
1572
1573         if (is->is_terminating || is->is_reconnecting)
1574                 return (EIO);
1575
1576         datalen = ids->ids_data_segment_len;
1577         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1578                 return (EINVAL);
1579         if (datalen > 0) {
1580                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1581                 error = copyin(ids->ids_data_segment, data, datalen);
1582                 if (error != 0) {
1583                         free(data, M_ISCSI);
1584                         return (error);
1585                 }
1586         }
1587
1588         ip = icl_pdu_new(is->is_conn, M_WAITOK);
1589         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1590         if (datalen > 0) {
1591                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1592                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1593                 free(data, M_ISCSI);
1594         }
1595         icl_pdu_queue(ip);
1596
1597         return (0);
1598 }
1599
1600 static int
1601 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1602     struct iscsi_daemon_receive *idr)
1603 {
1604         struct iscsi_session *is;
1605         struct icl_pdu *ip;
1606         void *data;
1607
1608         sx_slock(&sc->sc_lock);
1609         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1610                 if (is->is_id == idr->idr_session_id)
1611                         break;
1612         }
1613         if (is == NULL) {
1614                 sx_sunlock(&sc->sc_lock);
1615                 return (ESRCH);
1616         }
1617         sx_sunlock(&sc->sc_lock);
1618
1619         if (is->is_login_phase == false)
1620                 return (EBUSY);
1621
1622         ISCSI_SESSION_LOCK(is);
1623         while (is->is_login_pdu == NULL &&
1624             is->is_terminating == false &&
1625             is->is_reconnecting == false)
1626                 cv_wait(&is->is_login_cv, &is->is_lock);
1627         if (is->is_terminating || is->is_reconnecting) {
1628                 ISCSI_SESSION_UNLOCK(is);
1629                 return (EIO);
1630         }
1631         ip = is->is_login_pdu;
1632         is->is_login_pdu = NULL;
1633         ISCSI_SESSION_UNLOCK(is);
1634
1635         if (ip->ip_data_len > idr->idr_data_segment_len) {
1636                 icl_pdu_free(ip);
1637                 return (EMSGSIZE);
1638         }
1639
1640         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1641         if (ip->ip_data_len > 0) {
1642                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1643                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1644                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1645                 free(data, M_ISCSI);
1646         }
1647
1648         icl_pdu_free(ip);
1649
1650         return (0);
1651 }
1652 #endif /* ICL_KERNEL_PROXY */
1653
1654 static void
1655 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1656 {
1657         /*
1658          * Just make sure all the fields are null-terminated.
1659          *
1660          * XXX: This is not particularly secure.  We should
1661          *      create our own conf and then copy in relevant
1662          *      fields.
1663          */
1664         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1665         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1666         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1667         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1668         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1669         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1670         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1671         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1672         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1673 }
1674
1675 static bool
1676 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1677 {
1678
1679         if (isc->isc_initiator[0] == '\0') {
1680                 ISCSI_DEBUG("empty isc_initiator");
1681                 return (false);
1682         }
1683
1684         if (isc->isc_target_addr[0] == '\0') {
1685                 ISCSI_DEBUG("empty isc_target_addr");
1686                 return (false);
1687         }
1688
1689         if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1690                 ISCSI_DEBUG("non-empty isc_target for discovery session");
1691                 return (false);
1692         }
1693
1694         if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1695                 ISCSI_DEBUG("empty isc_target for non-discovery session");
1696                 return (false);
1697         }
1698
1699         return (true);
1700 }
1701
1702 static int
1703 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1704 {
1705         struct iscsi_session *is;
1706         const struct iscsi_session *is2;
1707         int error;
1708
1709         iscsi_sanitize_session_conf(&isa->isa_conf);
1710         if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1711                 return (EINVAL);
1712
1713         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1714         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1715
1716         sx_xlock(&sc->sc_lock);
1717
1718         /*
1719          * Prevent duplicates.
1720          */
1721         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1722                 if (!!is->is_conf.isc_discovery !=
1723                     !!is2->is_conf.isc_discovery)
1724                         continue;
1725
1726                 if (strcmp(is->is_conf.isc_target_addr,
1727                     is2->is_conf.isc_target_addr) != 0)
1728                         continue;
1729
1730                 if (is->is_conf.isc_discovery == 0 &&
1731                     strcmp(is->is_conf.isc_target,
1732                     is2->is_conf.isc_target) != 0)
1733                         continue;
1734
1735                 sx_xunlock(&sc->sc_lock);
1736                 free(is, M_ISCSI);
1737                 return (EBUSY);
1738         }
1739
1740         is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1741         is->is_conn->ic_receive = iscsi_receive_callback;
1742         is->is_conn->ic_error = iscsi_error_callback;
1743         is->is_conn->ic_prv0 = is;
1744         TAILQ_INIT(&is->is_outstanding);
1745         STAILQ_INIT(&is->is_postponed);
1746         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1747         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1748 #ifdef ICL_KERNEL_PROXY
1749         cv_init(&is->is_login_cv, "iscsi_login");
1750 #endif
1751
1752         is->is_softc = sc;
1753         sc->sc_last_session_id++;
1754         is->is_id = sc->sc_last_session_id;
1755         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1756         arc4rand(&is->is_isid[1], 5, 0);
1757         is->is_tsih = 0;
1758         callout_init(&is->is_callout, 1);
1759
1760         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1761         if (error != 0) {
1762                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1763                 sx_xunlock(&sc->sc_lock);
1764                 return (error);
1765         }
1766
1767         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1768         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1769
1770         /*
1771          * Trigger immediate reconnection.
1772          */
1773         ISCSI_SESSION_LOCK(is);
1774         is->is_waiting_for_iscsid = true;
1775         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1776         ISCSI_SESSION_UNLOCK(is);
1777         cv_signal(&sc->sc_cv);
1778
1779         sx_xunlock(&sc->sc_lock);
1780
1781         return (0);
1782 }
1783
1784 static bool
1785 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1786     unsigned int id2, const struct iscsi_session_conf *c2)
1787 {
1788
1789         if (id2 != 0 && id2 != id1)
1790                 return (false);
1791         if (c2->isc_target[0] != '\0' &&
1792             strcmp(c1->isc_target, c2->isc_target) != 0)
1793                 return (false);
1794         if (c2->isc_target_addr[0] != '\0' &&
1795             strcmp(c1->isc_target_addr, c2->isc_target_addr) != 0)
1796                 return (false);
1797         return (true);
1798 }
1799
1800 static int
1801 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1802     struct iscsi_session_remove *isr)
1803 {
1804         struct iscsi_session *is, *tmp;
1805         bool found = false;
1806
1807         iscsi_sanitize_session_conf(&isr->isr_conf);
1808
1809         sx_xlock(&sc->sc_lock);
1810         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1811                 ISCSI_SESSION_LOCK(is);
1812                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1813                     isr->isr_session_id, &isr->isr_conf)) {
1814                         found = true;
1815                         iscsi_session_logout(is);
1816                         iscsi_session_terminate(is);
1817                 }
1818                 ISCSI_SESSION_UNLOCK(is);
1819         }
1820         sx_xunlock(&sc->sc_lock);
1821
1822         if (!found)
1823                 return (ESRCH);
1824
1825         return (0);
1826 }
1827
1828 static int
1829 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1830 {
1831         int error;
1832         unsigned int i = 0;
1833         struct iscsi_session *is;
1834         struct iscsi_session_state iss;
1835
1836         sx_slock(&sc->sc_lock);
1837         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1838                 if (i >= isl->isl_nentries) {
1839                         sx_sunlock(&sc->sc_lock);
1840                         return (EMSGSIZE);
1841                 }
1842                 memset(&iss, 0, sizeof(iss));
1843                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1844                 iss.iss_id = is->is_id;
1845                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1846                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1847
1848                 if (is->is_conn->ic_header_crc32c)
1849                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1850                 else
1851                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1852
1853                 if (is->is_conn->ic_data_crc32c)
1854                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1855                 else
1856                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1857
1858                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1859                 iss.iss_immediate_data = is->is_immediate_data;
1860                 iss.iss_connected = is->is_connected;
1861
1862                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1863                 if (error != 0) {
1864                         sx_sunlock(&sc->sc_lock);
1865                         return (error);
1866                 }
1867                 i++;
1868         }
1869         sx_sunlock(&sc->sc_lock);
1870
1871         isl->isl_nentries = i;
1872
1873         return (0);
1874 }
1875
1876 static int
1877 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
1878     struct iscsi_session_modify *ism)
1879 {
1880         struct iscsi_session *is;
1881
1882         iscsi_sanitize_session_conf(&ism->ism_conf);
1883         if (iscsi_valid_session_conf(&ism->ism_conf) == false)
1884                 return (EINVAL);
1885
1886         sx_xlock(&sc->sc_lock);
1887         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1888                 ISCSI_SESSION_LOCK(is);
1889                 if (is->is_id == ism->ism_session_id)
1890                         break;
1891                 ISCSI_SESSION_UNLOCK(is);
1892         }
1893         if (is == NULL) {
1894                 sx_xunlock(&sc->sc_lock);
1895                 return (ESRCH);
1896         }
1897         sx_xunlock(&sc->sc_lock);
1898
1899         memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
1900         ISCSI_SESSION_UNLOCK(is);
1901
1902         iscsi_session_reconnect(is);
1903
1904         return (0);
1905 }
1906
1907 static int
1908 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1909     struct thread *td)
1910 {
1911         struct iscsi_softc *sc;
1912
1913         sc = dev->si_drv1;
1914
1915         switch (cmd) {
1916         case ISCSIDWAIT:
1917                 return (iscsi_ioctl_daemon_wait(sc,
1918                     (struct iscsi_daemon_request *)arg));
1919         case ISCSIDHANDOFF:
1920                 return (iscsi_ioctl_daemon_handoff(sc,
1921                     (struct iscsi_daemon_handoff *)arg));
1922         case ISCSIDFAIL:
1923                 return (iscsi_ioctl_daemon_fail(sc,
1924                     (struct iscsi_daemon_fail *)arg));
1925 #ifdef ICL_KERNEL_PROXY
1926         case ISCSIDCONNECT:
1927                 return (iscsi_ioctl_daemon_connect(sc,
1928                     (struct iscsi_daemon_connect *)arg));
1929         case ISCSIDSEND:
1930                 return (iscsi_ioctl_daemon_send(sc,
1931                     (struct iscsi_daemon_send *)arg));
1932         case ISCSIDRECEIVE:
1933                 return (iscsi_ioctl_daemon_receive(sc,
1934                     (struct iscsi_daemon_receive *)arg));
1935 #endif /* ICL_KERNEL_PROXY */
1936         case ISCSISADD:
1937                 return (iscsi_ioctl_session_add(sc,
1938                     (struct iscsi_session_add *)arg));
1939         case ISCSISREMOVE:
1940                 return (iscsi_ioctl_session_remove(sc,
1941                     (struct iscsi_session_remove *)arg));
1942         case ISCSISLIST:
1943                 return (iscsi_ioctl_session_list(sc,
1944                     (struct iscsi_session_list *)arg));
1945         case ISCSISMODIFY:
1946                 return (iscsi_ioctl_session_modify(sc,
1947                     (struct iscsi_session_modify *)arg));
1948         default:
1949                 return (EINVAL);
1950         }
1951 }
1952
1953 static uint64_t
1954 iscsi_encode_lun(uint32_t lun)
1955 {
1956         uint8_t encoded[8];
1957         uint64_t result;
1958
1959         memset(encoded, 0, sizeof(encoded));
1960
1961         if (lun < 256) {
1962                 /*
1963                  * Peripheral device addressing.
1964                  */
1965                 encoded[1] = lun;
1966         } else if (lun < 16384) {
1967                 /*
1968                  * Flat space addressing.
1969                  */
1970                 encoded[0] = 0x40;
1971                 encoded[0] |= (lun >> 8) & 0x3f;
1972                 encoded[1] = lun & 0xff;
1973         } else {
1974                 /*
1975                  * Extended flat space addressing.
1976                  */
1977                 encoded[0] = 0xd2;
1978                 encoded[1] = lun >> 16;
1979                 encoded[2] = lun >> 8;
1980                 encoded[3] = lun;
1981         }
1982
1983         memcpy(&result, encoded, sizeof(result));
1984         return (result);
1985 }
1986
1987 static struct iscsi_outstanding *
1988 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1989 {
1990         struct iscsi_outstanding *io;
1991
1992         ISCSI_SESSION_LOCK_ASSERT(is);
1993
1994         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1995                 if (io->io_initiator_task_tag == initiator_task_tag)
1996                         return (io);
1997         }
1998         return (NULL);
1999 }
2000
2001 static struct iscsi_outstanding *
2002 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
2003 {
2004         struct iscsi_outstanding *io;
2005
2006         ISCSI_SESSION_LOCK_ASSERT(is);
2007
2008         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2009                 if (io->io_ccb == ccb)
2010                         return (io);
2011         }
2012         return (NULL);
2013 }
2014
2015 static struct iscsi_outstanding *
2016 iscsi_outstanding_add(struct iscsi_session *is,
2017     uint32_t initiator_task_tag, union ccb *ccb)
2018 {
2019         struct iscsi_outstanding *io;
2020
2021         ISCSI_SESSION_LOCK_ASSERT(is);
2022
2023         KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
2024             ("initiator_task_tag 0x%x already added", initiator_task_tag));
2025
2026         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
2027         if (io == NULL) {
2028                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
2029                 return (NULL);
2030         }
2031         io->io_initiator_task_tag = initiator_task_tag;
2032         io->io_ccb = ccb;
2033         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
2034         return (io);
2035 }
2036
2037 static void
2038 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
2039 {
2040
2041         ISCSI_SESSION_LOCK_ASSERT(is);
2042
2043         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
2044         uma_zfree(iscsi_outstanding_zone, io);
2045 }
2046
2047 static void
2048 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
2049 {
2050         struct icl_pdu *request;
2051         struct iscsi_bhs_task_management_request *bhstmr;
2052         struct ccb_abort *cab = &ccb->cab;
2053         struct iscsi_outstanding *io, *aio;
2054
2055         ISCSI_SESSION_LOCK_ASSERT(is);
2056
2057 #if 0
2058         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2059 #else
2060         if (is->is_login_phase) {
2061                 ccb->ccb_h.status = CAM_REQ_ABORTED;
2062                 xpt_done(ccb);
2063                 return;
2064         }
2065 #endif
2066
2067         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
2068         if (aio == NULL) {
2069                 ccb->ccb_h.status = CAM_REQ_CMP;
2070                 xpt_done(ccb);
2071                 return;
2072         }
2073
2074         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2075         if (request == NULL) {
2076                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2077                 xpt_done(ccb);
2078                 return;
2079         }
2080
2081         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2082         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
2083         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
2084
2085         bhstmr->bhstmr_lun = iscsi_encode_lun(ccb->ccb_h.target_lun);
2086         bhstmr->bhstmr_initiator_task_tag = is->is_initiator_task_tag;
2087         is->is_initiator_task_tag++;
2088         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2089
2090         io = iscsi_outstanding_add(is, bhstmr->bhstmr_initiator_task_tag, NULL);
2091         if (io == NULL) {
2092                 icl_pdu_free(request);
2093                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2094                 xpt_done(ccb);
2095                 return;
2096         }
2097         io->io_datasn = aio->io_initiator_task_tag;
2098         iscsi_pdu_queue_locked(request);
2099 }
2100
2101 static void
2102 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2103 {
2104         struct icl_pdu *request;
2105         struct iscsi_bhs_scsi_command *bhssc;
2106         struct ccb_scsiio *csio;
2107         struct iscsi_outstanding *io;
2108         size_t len;
2109         int error;
2110
2111         ISCSI_SESSION_LOCK_ASSERT(is);
2112
2113 #if 0
2114         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2115 #else
2116         if (is->is_login_phase) {
2117                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2118                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2119                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2120                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2121                 }
2122                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2123                 xpt_done(ccb);
2124                 return;
2125         }
2126 #endif
2127
2128         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2129         if (request == NULL) {
2130                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2131                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2132                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2133                 }
2134                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2135                 xpt_done(ccb);
2136                 return;
2137         }
2138
2139         csio = &ccb->csio;
2140         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2141         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2142         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2143         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2144         case CAM_DIR_IN:
2145                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2146                 break;
2147         case CAM_DIR_OUT:
2148                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2149                 break;
2150         }
2151
2152         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2153                 switch (csio->tag_action) {
2154                 case MSG_HEAD_OF_Q_TAG:
2155                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2156                         break;
2157                 case MSG_ORDERED_Q_TAG:
2158                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2159                         break;
2160                 case MSG_ACA_TASK:
2161                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2162                         break;
2163                 case MSG_SIMPLE_Q_TAG:
2164                 default:
2165                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2166                         break;
2167                 }
2168         } else
2169                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2170
2171         bhssc->bhssc_lun = iscsi_encode_lun(csio->ccb_h.target_lun);
2172         bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
2173         is->is_initiator_task_tag++;
2174         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2175         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2176             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2177
2178         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2179                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2180         else
2181                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2182
2183         io = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
2184         if (io == NULL) {
2185                 icl_pdu_free(request);
2186                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2187                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2188                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2189                 }
2190                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2191                 xpt_done(ccb);
2192                 return;
2193         }
2194
2195         if (is->is_immediate_data &&
2196             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2197                 len = csio->dxfer_len;
2198                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2199                 if (len > is->is_first_burst_length) {
2200                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
2201                         len = is->is_first_burst_length;
2202                 }
2203                 if (len > is->is_max_data_segment_length) {
2204                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_max_data_segment_length);
2205                         len = is->is_max_data_segment_length;
2206                 }
2207
2208                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2209                 if (error != 0) {
2210                         icl_pdu_free(request);
2211                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2212                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2213                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2214                         }
2215                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2216                         xpt_done(ccb);
2217                         return;
2218                 }
2219         }
2220         iscsi_pdu_queue_locked(request);
2221 }
2222
2223 static void
2224 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2225 {
2226         struct iscsi_session *is;
2227
2228         is = cam_sim_softc(sim);
2229
2230         ISCSI_SESSION_LOCK_ASSERT(is);
2231
2232         if (is->is_terminating ||
2233             (is->is_connected == false && fail_on_disconnection)) {
2234                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2235                 xpt_done(ccb);
2236                 return;
2237         }
2238
2239         switch (ccb->ccb_h.func_code) {
2240         case XPT_PATH_INQ:
2241         {
2242                 struct ccb_pathinq *cpi = &ccb->cpi;
2243
2244                 cpi->version_num = 1;
2245                 cpi->hba_inquiry = PI_TAG_ABLE;
2246                 cpi->target_sprt = 0;
2247                 //cpi->hba_misc = PIM_NOBUSRESET;
2248                 cpi->hba_misc = 0;
2249                 cpi->hba_eng_cnt = 0;
2250                 cpi->max_target = 0;
2251                 cpi->max_lun = 255;
2252                 //cpi->initiator_id = 0; /* XXX */
2253                 cpi->initiator_id = 64; /* XXX */
2254                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2255                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2256                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2257                 cpi->unit_number = cam_sim_unit(sim);
2258                 cpi->bus_id = cam_sim_bus(sim);
2259                 cpi->base_transfer_speed = 150000; /* XXX */
2260                 cpi->transport = XPORT_ISCSI;
2261                 cpi->transport_version = 0;
2262                 cpi->protocol = PROTO_SCSI;
2263                 cpi->protocol_version = SCSI_REV_SPC3;
2264                 cpi->maxio = MAXPHYS;
2265                 cpi->ccb_h.status = CAM_REQ_CMP;
2266                 break;
2267         }
2268         case XPT_GET_TRAN_SETTINGS:
2269         {
2270                 struct ccb_trans_settings       *cts;
2271                 struct ccb_trans_settings_scsi  *scsi;
2272
2273                 cts = &ccb->cts;
2274                 scsi = &cts->proto_specific.scsi;
2275
2276                 cts->protocol = PROTO_SCSI;
2277                 cts->protocol_version = SCSI_REV_SPC3;
2278                 cts->transport = XPORT_ISCSI;
2279                 cts->transport_version = 0;
2280                 scsi->valid = CTS_SCSI_VALID_TQ;
2281                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2282                 cts->ccb_h.status = CAM_REQ_CMP;
2283                 break;
2284         }
2285         case XPT_CALC_GEOMETRY:
2286                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2287                 ccb->ccb_h.status = CAM_REQ_CMP;
2288                 break;
2289 #if 0
2290         /*
2291          * XXX: What's the point?
2292          */
2293         case XPT_RESET_BUS:
2294         case XPT_TERM_IO:
2295                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2296                 ccb->ccb_h.status = CAM_REQ_CMP;
2297                 break;
2298 #endif
2299         case XPT_ABORT:
2300                 iscsi_action_abort(is, ccb);
2301                 return;
2302         case XPT_SCSI_IO:
2303                 iscsi_action_scsiio(is, ccb);
2304                 return;
2305         default:
2306 #if 0
2307                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2308 #endif
2309                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2310                 break;
2311         }
2312         xpt_done(ccb);
2313 }
2314
2315 static void
2316 iscsi_poll(struct cam_sim *sim)
2317 {
2318
2319         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2320 }
2321
2322 static void
2323 iscsi_shutdown(struct iscsi_softc *sc)
2324 {
2325         struct iscsi_session *is;
2326
2327         ISCSI_DEBUG("removing all sessions due to shutdown");
2328
2329         sx_slock(&sc->sc_lock);
2330         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2331                 iscsi_session_terminate(is);
2332         sx_sunlock(&sc->sc_lock);
2333 }
2334
2335 static int
2336 iscsi_load(void)
2337 {
2338         int error;
2339
2340         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2341         sx_init(&sc->sc_lock, "iscsi");
2342         TAILQ_INIT(&sc->sc_sessions);
2343         cv_init(&sc->sc_cv, "iscsi_cv");
2344
2345         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2346             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2347             UMA_ALIGN_PTR, 0);
2348
2349         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2350             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2351         if (error != 0) {
2352                 ISCSI_WARN("failed to create device node, error %d", error);
2353                 return (error);
2354         }
2355         sc->sc_cdev->si_drv1 = sc;
2356
2357         /*
2358          * Note that this needs to get run before dashutdown().  Otherwise,
2359          * when rebooting with iSCSI session with outstanding requests,
2360          * but disconnected, dashutdown() will hang on cam_periph_runccb().
2361          */
2362         sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2363             iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2364
2365         return (0);
2366 }
2367
2368 static int
2369 iscsi_unload(void)
2370 {
2371         struct iscsi_session *is, *tmp;
2372
2373         if (sc->sc_cdev != NULL) {
2374                 ISCSI_DEBUG("removing device node");
2375                 destroy_dev(sc->sc_cdev);
2376                 ISCSI_DEBUG("device node removed");
2377         }
2378
2379         if (sc->sc_shutdown_eh != NULL)
2380                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2381
2382         sx_slock(&sc->sc_lock);
2383         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2384                 iscsi_session_terminate(is);
2385         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2386                 ISCSI_DEBUG("waiting for sessions to terminate");
2387                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2388         }
2389         ISCSI_DEBUG("all sessions terminated");
2390         sx_sunlock(&sc->sc_lock);
2391
2392         uma_zdestroy(iscsi_outstanding_zone);
2393         sx_destroy(&sc->sc_lock);
2394         cv_destroy(&sc->sc_cv);
2395         free(sc, M_ISCSI);
2396         return (0);
2397 }
2398
2399 static int
2400 iscsi_quiesce(void)
2401 {
2402         sx_slock(&sc->sc_lock);
2403         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2404                 sx_sunlock(&sc->sc_lock);
2405                 return (EBUSY);
2406         }
2407         sx_sunlock(&sc->sc_lock);
2408         return (0);
2409 }
2410
2411 static int
2412 iscsi_modevent(module_t mod, int what, void *arg)
2413 {
2414         int error;
2415
2416         switch (what) {
2417         case MOD_LOAD:
2418                 error = iscsi_load();
2419                 break;
2420         case MOD_UNLOAD:
2421                 error = iscsi_unload();
2422                 break;
2423         case MOD_QUIESCE:
2424                 error = iscsi_quiesce();
2425                 break;
2426         default:
2427                 error = EINVAL;
2428                 break;
2429         }
2430         return (error);
2431 }
2432
2433 moduledata_t iscsi_data = {
2434         "iscsi",
2435         iscsi_modevent,
2436         0
2437 };
2438
2439 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2440 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2441 MODULE_DEPEND(iscsi, icl, 1, 1, 1);