sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #include <sys/param.h>
  33 #include <sys/condvar.h>
  34 #include <sys/conf.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/file.h>
  37 #include <sys/kernel.h>
  38 #include <sys/kthread.h>
  39 #include <sys/lock.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mutex.h>
  42 #include <sys/module.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/systm.h>
  45 #include <sys/sx.h>
  46 #include <vm/uma.h>
  47
  48 #include <cam/cam.h>
  49 #include <cam/cam_ccb.h>
  50 #include <cam/cam_xpt.h>
  51 #include <cam/cam_debug.h>
  52 #include <cam/cam_sim.h>
  53 #include <cam/cam_xpt_sim.h>
  54 #include <cam/cam_xpt_periph.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/scsi/scsi_all.h>
  57 #include <cam/scsi/scsi_message.h>
  58
  59 #include "iscsi_ioctl.h"
  60 #include "iscsi.h"
  61 #include "icl.h"
  62 #include "iscsi_proto.h"
  63
  64 #ifdef ICL_KERNEL_PROXY
  65 #include <sys/socketvar.h>
  66 #endif
  67
  68 #ifdef ICL_KERNEL_PROXY
  69 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  70 #endif
  71
  72 /*
  73  * XXX: This is global so the iscsi_unload() can access it.
  74  *      Think about how to do this properly.
  75  */
  76 static struct iscsi_softc       *sc;
  77
  78 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  79 static int debug = 1;
  80 TUNABLE_INT("kern.iscsi.debug", &debug);
  81 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  82     &debug, 0, "Enable debug messages");
  83 static int ping_timeout = 5;
  84 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
  85 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  86     0, "Timeout for ping (NOP-Out) requests, in seconds");
  87 static int iscsid_timeout = 60;
  88 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
  89 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  90     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  91 static int login_timeout = 60;
  92 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
  93 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
  94     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  95 static int maxtags = 255;
  96 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
  97 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
  98     0, "Max number of IO requests queued");
  99 static int fail_on_disconnection = 0;
 100 TUNABLE_INT("kern.iscsi.fail_on_disconnection", &fail_on_disconnection);
 101 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
 102     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
 103
 104 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 105 static uma_zone_t iscsi_outstanding_zone;
 106
 107 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 108 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 109
 110 #define ISCSI_DEBUG(X, ...)                                             \
 111         do {                                                            \
 112                 if (debug > 1)                                          \
 113                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 114         } while (0)
 115
 116 #define ISCSI_WARN(X, ...)                                              \
 117         do {                                                            \
 118                 if (debug > 0) {                                        \
 119                         printf("WARNING: %s: " X "\n",                  \
 120                             __func__, ## __VA_ARGS__);                  \
 121                 }                                                       \
 122         } while (0)
 123
 124 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 125         do {                                                            \
 126                 if (debug > 1) {                                        \
 127                         printf("%s: %s (%s): " X "\n",                  \
 128                             __func__, S->is_conf.isc_target_addr,       \
 129                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 130                 }                                                       \
 131         } while (0)
 132
 133 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 134         do {                                                            \
 135                 if (debug > 0) {                                        \
 136                         printf("WARNING: %s (%s): " X "\n",             \
 137                             S->is_conf.isc_target_addr,                 \
 138                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 139                 }                                                       \
 140         } while (0)
 141
 142 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 143 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 144 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 145
 146 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 147                     int mode, struct thread *td);
 148
 149 static struct cdevsw iscsi_cdevsw = {
 150      .d_version = D_VERSION,
 151      .d_ioctl   = iscsi_ioctl,
 152      .d_name    = "iscsi",
 153 };
 154
 155 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 156 static void     iscsi_pdu_queue(struct icl_pdu *request);
 157 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 158 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 159 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 160 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
 161 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 162 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 163 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 164 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 165 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 166 static void     iscsi_session_reconnect(struct iscsi_session *is);
 167 static void     iscsi_session_terminate(struct iscsi_session *is);
 168 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 169 static void     iscsi_poll(struct cam_sim *sim);
 170 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 171                     uint32_t initiator_task_tag);
 172 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
 173                     uint32_t initiator_task_tag, union ccb *ccb);
 174 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 175                     struct iscsi_outstanding *io);
 176
 177 static bool
 178 iscsi_pdu_prepare(struct icl_pdu *request)
 179 {
 180         struct iscsi_session *is;
 181         struct iscsi_bhs_scsi_command *bhssc;
 182
 183         is = PDU_SESSION(request);
 184
 185         ISCSI_SESSION_LOCK_ASSERT(is);
 186
 187         /*
 188          * We're only using fields common for all the request
 189          * (initiator -> target) PDUs.
 190          */
 191         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 192
 193         /*
 194          * Data-Out PDU does not contain CmdSN.
 195          */
 196         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 197                 if (is->is_cmdsn > is->is_maxcmdsn &&
 198                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 199                         /*
 200                          * Current MaxCmdSN prevents us from sending any more
 201                          * SCSI Command PDUs to the target; postpone the PDU.
 202                          * It will get resent by either iscsi_pdu_queue(),
 203                          * or by maintenance thread.
 204                          */
 205 #if 0
 206                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %d, ExpCmdSN %d, MaxCmdSN %d, opcode 0x%x",
 207                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn, bhssc->bhssc_opcode);
 208 #endif
 209                         return (true);
 210                 }
 211                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 212                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 213                         is->is_cmdsn++;
 214         }
 215         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 216
 217         return (false);
 218 }
 219
 220 static void
 221 iscsi_session_send_postponed(struct iscsi_session *is)
 222 {
 223         struct icl_pdu *request;
 224         bool postpone;
 225
 226         ISCSI_SESSION_LOCK_ASSERT(is);
 227
 228         while (!STAILQ_EMPTY(&is->is_postponed)) {
 229                 request = STAILQ_FIRST(&is->is_postponed);
 230                 postpone = iscsi_pdu_prepare(request);
 231                 if (postpone)
 232                         break;
 233                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 234                 icl_pdu_queue(request);
 235         }
 236 }
 237
 238 static void
 239 iscsi_pdu_queue_locked(struct icl_pdu *request)
 240 {
 241         struct iscsi_session *is;
 242         bool postpone;
 243
 244         is = PDU_SESSION(request);
 245         ISCSI_SESSION_LOCK_ASSERT(is);
 246         iscsi_session_send_postponed(is);
 247         postpone = iscsi_pdu_prepare(request);
 248         if (postpone) {
 249                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 250                 return;
 251         }
 252         icl_pdu_queue(request);
 253 }
 254
 255 static void
 256 iscsi_pdu_queue(struct icl_pdu *request)
 257 {
 258         struct iscsi_session *is;
 259
 260         is = PDU_SESSION(request);
 261         ISCSI_SESSION_LOCK(is);
 262         iscsi_pdu_queue_locked(request);
 263         ISCSI_SESSION_UNLOCK(is);
 264 }
 265
 266 static void
 267 iscsi_session_logout(struct iscsi_session *is)
 268 {
 269         struct icl_pdu *request;
 270         struct iscsi_bhs_logout_request *bhslr;
 271
 272         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 273         if (request == NULL)
 274                 return;
 275
 276         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 277         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 278         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 279         iscsi_pdu_queue_locked(request);
 280 }
 281
 282 static void
 283 iscsi_session_terminate_task(struct iscsi_session *is,
 284     struct iscsi_outstanding *io, bool requeue)
 285 {
 286
 287         if (io->io_ccb != NULL) {
 288                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
 289                 if (requeue)
 290                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 291                 else
 292                         io->io_ccb->ccb_h.status |= CAM_REQ_ABORTED;
 293                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 294                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 295                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 296                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 297                 }
 298                 xpt_done(io->io_ccb);
 299         }
 300         iscsi_outstanding_remove(is, io);
 301 }
 302
 303 static void
 304 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 305 {
 306         struct iscsi_outstanding *io, *tmp;
 307
 308         ISCSI_SESSION_LOCK_ASSERT(is);
 309
 310         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 311                 iscsi_session_terminate_task(is, io, requeue);
 312         }
 313 }
 314
 315 static void
 316 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 317 {
 318         struct icl_pdu *pdu;
 319
 320         ISCSI_SESSION_LOCK_ASSERT(is);
 321
 322         /*
 323          * Don't queue any new PDUs.
 324          */
 325         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 326                 ISCSI_SESSION_DEBUG(is, "freezing");
 327                 xpt_freeze_simq(is->is_sim, 1);
 328                 is->is_simq_frozen = true;
 329         }
 330
 331         /*
 332          * Remove postponed PDUs.
 333          */
 334         while (!STAILQ_EMPTY(&is->is_postponed)) {
 335                 pdu = STAILQ_FIRST(&is->is_postponed);
 336                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 337                 icl_pdu_free(pdu);
 338         }
 339
 340         if (destroy_sim == false) {
 341                 /*
 342                  * Terminate SCSI tasks, asking CAM to requeue them.
 343                  */
 344                 iscsi_session_terminate_tasks(is, true);
 345                 return;
 346         }
 347
 348         iscsi_session_terminate_tasks(is, false);
 349
 350         if (is->is_sim == NULL)
 351                 return;
 352
 353         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 354         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 355
 356         if (is->is_simq_frozen) {
 357                 xpt_release_simq(is->is_sim, 1);
 358                 is->is_simq_frozen = false;
 359         }
 360
 361         xpt_free_path(is->is_path);
 362         is->is_path = NULL;
 363         xpt_bus_deregister(cam_sim_path(is->is_sim));
 364         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 365         is->is_sim = NULL;
 366         is->is_devq = NULL;
 367 }
 368
 369 static void
 370 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 371 {
 372
 373         icl_conn_shutdown(is->is_conn);
 374         icl_conn_close(is->is_conn);
 375
 376         ISCSI_SESSION_LOCK(is);
 377
 378         is->is_connected = false;
 379         is->is_reconnecting = false;
 380         is->is_login_phase = false;
 381
 382 #ifdef ICL_KERNEL_PROXY
 383         if (is->is_login_pdu != NULL) {
 384                 icl_pdu_free(is->is_login_pdu);
 385                 is->is_login_pdu = NULL;
 386         }
 387         cv_signal(&is->is_login_cv);
 388 #endif
 389
 390         if (fail_on_disconnection) {
 391                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
 392                 iscsi_session_cleanup(is, true);
 393         } else {
 394                 iscsi_session_cleanup(is, false);
 395         }
 396
 397         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 398             ("destroying session with active tasks"));
 399         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 400             ("destroying session with postponed PDUs"));
 401
 402         /*
 403          * Request immediate reconnection from iscsid(8).
 404          */
 405         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 406         is->is_waiting_for_iscsid = true;
 407         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 408         is->is_timeout = 0;
 409         ISCSI_SESSION_UNLOCK(is);
 410         cv_signal(&is->is_softc->sc_cv);
 411 }
 412
 413 static void
 414 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 415 {
 416         struct iscsi_softc *sc;
 417
 418         sc = is->is_softc;
 419         sx_xlock(&sc->sc_lock);
 420         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 421         sx_xunlock(&sc->sc_lock);
 422
 423         icl_conn_close(is->is_conn);
 424
 425         ISCSI_SESSION_LOCK(is);
 426
 427         KASSERT(is->is_terminating, ("is_terminating == false"));
 428
 429 #ifdef ICL_KERNEL_PROXY
 430         if (is->is_login_pdu != NULL) {
 431                 icl_pdu_free(is->is_login_pdu);
 432                 is->is_login_pdu = NULL;
 433         }
 434         cv_signal(&is->is_login_cv);
 435 #endif
 436
 437         callout_drain(&is->is_callout);
 438
 439         iscsi_session_cleanup(is, true);
 440
 441         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 442             ("destroying session with active tasks"));
 443         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 444             ("destroying session with postponed PDUs"));
 445
 446         ISCSI_SESSION_UNLOCK(is);
 447
 448         icl_conn_free(is->is_conn);
 449         mtx_destroy(&is->is_lock);
 450         cv_destroy(&is->is_maintenance_cv);
 451 #ifdef ICL_KERNEL_PROXY
 452         cv_destroy(&is->is_login_cv);
 453 #endif
 454         ISCSI_SESSION_DEBUG(is, "terminated");
 455         free(is, M_ISCSI);
 456
 457         /*
 458          * The iscsi_unload() routine might be waiting.
 459          */
 460         cv_signal(&sc->sc_cv);
 461 }
 462
 463 static void
 464 iscsi_maintenance_thread(void *arg)
 465 {
 466         struct iscsi_session *is;
 467
 468         is = arg;
 469
 470         for (;;) {
 471                 ISCSI_SESSION_LOCK(is);
 472                 if (is->is_reconnecting == false &&
 473                     is->is_terminating == false &&
 474                     STAILQ_EMPTY(&is->is_postponed))
 475                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 476
 477                 if (is->is_reconnecting) {
 478                         ISCSI_SESSION_UNLOCK(is);
 479                         iscsi_maintenance_thread_reconnect(is);
 480                         continue;
 481                 }
 482
 483                 if (is->is_terminating) {
 484                         ISCSI_SESSION_UNLOCK(is);
 485                         iscsi_maintenance_thread_terminate(is);
 486                         kthread_exit();
 487                         return;
 488                 }
 489
 490                 iscsi_session_send_postponed(is);
 491                 ISCSI_SESSION_UNLOCK(is);
 492         }
 493 }
 494
 495 static void
 496 iscsi_session_reconnect(struct iscsi_session *is)
 497 {
 498
 499         /*
 500          * XXX: We can't use locking here, because
 501          *      it's being called from various contexts.
 502          *      Hope it doesn't break anything.
 503          */
 504         if (is->is_reconnecting)
 505                 return;
 506
 507         is->is_reconnecting = true;
 508         cv_signal(&is->is_maintenance_cv);
 509 }
 510
 511 static void
 512 iscsi_session_terminate(struct iscsi_session *is)
 513 {
 514         if (is->is_terminating)
 515                 return;
 516
 517         is->is_terminating = true;
 518
 519 #if 0
 520         iscsi_session_logout(is);
 521 #endif
 522         cv_signal(&is->is_maintenance_cv);
 523 }
 524
 525 static void
 526 iscsi_callout(void *context)
 527 {
 528         struct icl_pdu *request;
 529         struct iscsi_bhs_nop_out *bhsno;
 530         struct iscsi_session *is;
 531         bool reconnect_needed = false;
 532
 533         is = context;
 534
 535         if (is->is_terminating)
 536                 return;
 537
 538         callout_schedule(&is->is_callout, 1 * hz);
 539
 540         ISCSI_SESSION_LOCK(is);
 541         is->is_timeout++;
 542
 543         if (is->is_waiting_for_iscsid) {
 544                 if (is->is_timeout > iscsid_timeout) {
 545                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 546                             "for %d seconds; reconnecting",
 547                             is->is_timeout);
 548                         reconnect_needed = true;
 549                 }
 550                 goto out;
 551         }
 552
 553         if (is->is_login_phase) {
 554                 if (is->is_timeout > login_timeout) {
 555                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 556                             "reconnecting", is->is_timeout);
 557                         reconnect_needed = true;
 558                 }
 559                 goto out;
 560         }
 561
 562         if (is->is_timeout >= ping_timeout) {
 563                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 564                     "reconnecting", ping_timeout);
 565                 reconnect_needed = true;
 566                 goto out;
 567         }
 568
 569         ISCSI_SESSION_UNLOCK(is);
 570
 571         /*
 572          * If the ping was reset less than one second ago - which means
 573          * that we've received some PDU during the last second - assume
 574          * the traffic flows correctly and don't bother sending a NOP-Out.
 575          *
 576          * (It's 2 - one for one second, and one for incrementing is_timeout
 577          * earlier in this routine.)
 578          */
 579         if (is->is_timeout < 2)
 580                 return;
 581
 582         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 583         if (request == NULL) {
 584                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 585                 return;
 586         }
 587         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 588         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 589             ISCSI_BHS_OPCODE_IMMEDIATE;
 590         bhsno->bhsno_flags = 0x80;
 591         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 592         iscsi_pdu_queue(request);
 593         return;
 594
 595 out:
 596         ISCSI_SESSION_UNLOCK(is);
 597
 598         if (reconnect_needed)
 599                 iscsi_session_reconnect(is);
 600 }
 601
 602 static void
 603 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 604 {
 605         const struct iscsi_bhs_data_in *bhsdi;
 606         struct iscsi_session *is;
 607         uint32_t expcmdsn, maxcmdsn;
 608
 609         is = PDU_SESSION(response);
 610
 611         ISCSI_SESSION_LOCK_ASSERT(is);
 612
 613         /*
 614          * We're only using fields common for all the response
 615          * (target -> initiator) PDUs.
 616          */
 617         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 618         /*
 619          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 620          * and Residual Count only have meaningful content if the S bit
 621          * is set to 1", so we also need to check the bit specific for
 622          * Data-In PDU.
 623          */
 624         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 625             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 626                 if (ntohl(bhsdi->bhsdi_statsn) < is->is_statsn) {
 627                         ISCSI_SESSION_WARN(is,
 628                             "PDU StatSN %d >= session StatSN %d, opcode 0x%x",
 629                             is->is_statsn, ntohl(bhsdi->bhsdi_statsn),
 630                             bhsdi->bhsdi_opcode);
 631                 }
 632                 is->is_statsn = ntohl(bhsdi->bhsdi_statsn);
 633         }
 634
 635         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 636         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 637
 638         /*
 639          * XXX: Compare using Serial Arithmetic Sense.
 640          */
 641         if (maxcmdsn + 1 < expcmdsn) {
 642                 ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d + 1 < PDU ExpCmdSN %d; ignoring",
 643                     maxcmdsn, expcmdsn);
 644         } else {
 645                 if (maxcmdsn > is->is_maxcmdsn) {
 646                         is->is_maxcmdsn = maxcmdsn;
 647
 648                         /*
 649                          * Command window increased; kick the maintanance thread
 650                          * to send out postponed commands.
 651                          */
 652                         if (!STAILQ_EMPTY(&is->is_postponed))
 653                                 cv_signal(&is->is_maintenance_cv);
 654                 } else if (maxcmdsn < is->is_maxcmdsn) {
 655                         ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d < session MaxCmdSN %d; ignoring",
 656                             maxcmdsn, is->is_maxcmdsn);
 657                 }
 658
 659                 if (expcmdsn > is->is_expcmdsn) {
 660                         is->is_expcmdsn = expcmdsn;
 661                 } else if (expcmdsn < is->is_expcmdsn) {
 662                         ISCSI_SESSION_DEBUG(is, "PDU ExpCmdSN %d < session ExpCmdSN %d; ignoring",
 663                             expcmdsn, is->is_expcmdsn);
 664                 }
 665         }
 666
 667         /*
 668          * Every incoming PDU - not just NOP-In - resets the ping timer.
 669          * The purpose of the timeout is to reset the connection when it stalls;
 670          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 671          * in some queue.
 672          */
 673         is->is_timeout = 0;
 674 }
 675
 676 static void
 677 iscsi_receive_callback(struct icl_pdu *response)
 678 {
 679         struct iscsi_session *is;
 680
 681         is = PDU_SESSION(response);
 682
 683         ISCSI_SESSION_LOCK(is);
 684
 685 #ifdef ICL_KERNEL_PROXY
 686         if (is->is_login_phase) {
 687                 if (is->is_login_pdu == NULL)
 688                         is->is_login_pdu = response;
 689                 else
 690                         icl_pdu_free(response);
 691                 ISCSI_SESSION_UNLOCK(is);
 692                 cv_signal(&is->is_login_cv);
 693                 return;
 694         }
 695 #endif
 696
 697         iscsi_pdu_update_statsn(response);
 698
 699         /*
 700          * The handling routine is responsible for freeing the PDU
 701          * when it's no longer needed.
 702          */
 703         switch (response->ip_bhs->bhs_opcode) {
 704         case ISCSI_BHS_OPCODE_NOP_IN:
 705                 iscsi_pdu_handle_nop_in(response);
 706                 break;
 707         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 708                 iscsi_pdu_handle_scsi_response(response);
 709                 break;
 710         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
 711                 iscsi_pdu_handle_task_response(response);
 712                 break;
 713         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 714                 iscsi_pdu_handle_data_in(response);
 715                 break;
 716         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 717                 iscsi_pdu_handle_logout_response(response);
 718                 break;
 719         case ISCSI_BHS_OPCODE_R2T:
 720                 iscsi_pdu_handle_r2t(response);
 721                 break;
 722         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 723                 iscsi_pdu_handle_async_message(response);
 724                 break;
 725         case ISCSI_BHS_OPCODE_REJECT:
 726                 iscsi_pdu_handle_reject(response);
 727                 break;
 728         default:
 729                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 730                     "opcode 0x%x; reconnecting",
 731                     response->ip_bhs->bhs_opcode);
 732                 iscsi_session_reconnect(is);
 733                 icl_pdu_free(response);
 734         }
 735
 736         ISCSI_SESSION_UNLOCK(is);
 737 }
 738
 739 static void
 740 iscsi_error_callback(struct icl_conn *ic)
 741 {
 742         struct iscsi_session *is;
 743
 744         is = CONN_SESSION(ic);
 745
 746         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 747         iscsi_session_reconnect(is);
 748 }
 749
 750 static void
 751 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 752 {
 753         struct iscsi_session *is;
 754         struct iscsi_bhs_nop_out *bhsno;
 755         struct iscsi_bhs_nop_in *bhsni;
 756         struct icl_pdu *request;
 757         void *data = NULL;
 758         size_t datasize;
 759         int error;
 760
 761         is = PDU_SESSION(response);
 762         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 763
 764         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 765                 /*
 766                  * Nothing to do; iscsi_pdu_update_statsn() already
 767                  * zeroed the timeout.
 768                  */
 769                 icl_pdu_free(response);
 770                 return;
 771         }
 772
 773         datasize = icl_pdu_data_segment_length(response);
 774         if (datasize > 0) {
 775                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 776                 if (data == NULL) {
 777                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 778                             "reconnecting");
 779                         icl_pdu_free(response);
 780                         iscsi_session_reconnect(is);
 781                         return;
 782                 }
 783                 icl_pdu_get_data(response, 0, data, datasize);
 784         }
 785
 786         request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
 787         if (request == NULL) {
 788                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 789                     "reconnecting");
 790                 free(data, M_ISCSI);
 791                 icl_pdu_free(response);
 792                 iscsi_session_reconnect(is);
 793                 return;
 794         }
 795         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 796         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 797             ISCSI_BHS_OPCODE_IMMEDIATE;
 798         bhsno->bhsno_flags = 0x80;
 799         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 800         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 801         if (datasize > 0) {
 802                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 803                 if (error != 0) {
 804                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 805                             "reconnecting");
 806                         free(data, M_ISCSI);
 807                         icl_pdu_free(request);
 808                         icl_pdu_free(response);
 809                         iscsi_session_reconnect(is);
 810                         return;
 811                 }
 812                 free(data, M_ISCSI);
 813         }
 814
 815         icl_pdu_free(response);
 816         iscsi_pdu_queue_locked(request);
 817 }
 818
 819 static void
 820 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 821 {
 822         struct iscsi_bhs_scsi_response *bhssr;
 823         struct iscsi_outstanding *io;
 824         struct iscsi_session *is;
 825         struct ccb_scsiio *csio;
 826         size_t data_segment_len;
 827         uint16_t sense_len;
 828
 829         is = PDU_SESSION(response);
 830
 831         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 832         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 833         if (io == NULL || io->io_ccb == NULL) {
 834                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 835                 icl_pdu_free(response);
 836                 iscsi_session_reconnect(is);
 837                 return;
 838         }
 839
 840         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 841                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 842                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 843                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 844                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 845                 }
 846                 io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 847         } else if (bhssr->bhssr_status == 0) {
 848                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
 849         } else {
 850                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 851                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 852                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 853                 }
 854                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 855                 io->io_ccb->csio.scsi_status = bhssr->bhssr_status;
 856         }
 857
 858         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
 859                 ISCSI_SESSION_WARN(is, "target indicated residual overflow");
 860                 icl_pdu_free(response);
 861                 iscsi_session_reconnect(is);
 862                 return;
 863         }
 864
 865         csio = &io->io_ccb->csio;
 866
 867         data_segment_len = icl_pdu_data_segment_length(response);
 868         if (data_segment_len > 0) {
 869                 if (data_segment_len < sizeof(sense_len)) {
 870                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 871                             data_segment_len);
 872                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 873                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 874                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 875                         }
 876                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 877                         goto out;
 878                 }
 879                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 880                 sense_len = ntohs(sense_len);
 881 #if 0
 882                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 883                     sense_len, data_segment_len);
 884 #endif
 885                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 886                         ISCSI_SESSION_WARN(is, "truncated data segment "
 887                             "(%zd bytes, should be %zd)",
 888                             data_segment_len, sizeof(sense_len) + sense_len);
 889                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 890                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 891                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 892                         }
 893                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 894                         goto out;
 895                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 896                         ISCSI_SESSION_WARN(is, "oversize data segment "
 897                             "(%zd bytes, should be %zd)",
 898                             data_segment_len, sizeof(sense_len) + sense_len);
 899                 if (sense_len > csio->sense_len) {
 900                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 901                             sense_len, csio->sense_len);
 902                         sense_len = csio->sense_len;
 903                 }
 904                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 905                 csio->sense_resid = csio->sense_len - sense_len;
 906                 io->io_ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 907         }
 908
 909 out:
 910         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 911                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 912
 913         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 914                 KASSERT(io->io_received <= csio->dxfer_len,
 915                     ("io->io_received > csio->dxfer_len"));
 916                 if (io->io_received < csio->dxfer_len) {
 917                         if (csio->resid != csio->dxfer_len - io->io_received) {
 918                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 919                                     "target indicates %d, we calculated %zd",
 920                                     csio->resid,
 921                                     csio->dxfer_len - io->io_received);
 922                         }
 923                         csio->resid = csio->dxfer_len - io->io_received;
 924                 }
 925         }
 926
 927         xpt_done(io->io_ccb);
 928         iscsi_outstanding_remove(is, io);
 929         icl_pdu_free(response);
 930 }
 931
 932 static void
 933 iscsi_pdu_handle_task_response(struct icl_pdu *response)
 934 {
 935         struct iscsi_bhs_task_management_response *bhstmr;
 936         struct iscsi_outstanding *io, *aio;
 937         struct iscsi_session *is;
 938
 939         is = PDU_SESSION(response);
 940
 941         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
 942         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
 943         if (io == NULL || io->io_ccb != NULL) {
 944                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
 945                     bhstmr->bhstmr_initiator_task_tag);
 946                 icl_pdu_free(response);
 947                 iscsi_session_reconnect(is);
 948                 return;
 949         }
 950
 951         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
 952                 ISCSI_SESSION_WARN(is, "task response 0x%x",
 953                     bhstmr->bhstmr_response);
 954         } else {
 955                 aio = iscsi_outstanding_find(is, io->io_datasn);
 956                 if (aio != NULL && aio->io_ccb != NULL)
 957                         iscsi_session_terminate_task(is, aio, false);
 958         }
 959
 960         iscsi_outstanding_remove(is, io);
 961         icl_pdu_free(response);
 962 }
 963
 964 static void
 965 iscsi_pdu_handle_data_in(struct icl_pdu *response)
 966 {
 967         struct iscsi_bhs_data_in *bhsdi;
 968         struct iscsi_outstanding *io;
 969         struct iscsi_session *is;
 970         struct ccb_scsiio *csio;
 971         size_t data_segment_len;
 972
 973         is = PDU_SESSION(response);
 974         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
 975         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
 976         if (io == NULL || io->io_ccb == NULL) {
 977                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
 978                 icl_pdu_free(response);
 979                 iscsi_session_reconnect(is);
 980                 return;
 981         }
 982
 983         data_segment_len = icl_pdu_data_segment_length(response);
 984         if (data_segment_len == 0) {
 985                 /*
 986                  * "The sending of 0 length data segments should be avoided,
 987                  * but initiators and targets MUST be able to properly receive
 988                  * 0 length data segments."
 989                  */
 990                 icl_pdu_free(response);
 991                 return;
 992         }
 993
 994         /*
 995          * We need to track this for security reasons - without it, malicious target
 996          * could respond to SCSI READ without sending Data-In PDUs, which would result
 997          * in read operation on the initiator side returning random kernel data.
 998          */
 999         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1000                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1001                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1002                 icl_pdu_free(response);
1003                 iscsi_session_reconnect(is);
1004                 return;
1005         }
1006
1007         csio = &io->io_ccb->csio;
1008
1009         if (io->io_received + data_segment_len > csio->dxfer_len) {
1010                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1011                     "at offset %zd, buffer is %d)",
1012                     data_segment_len, io->io_received, csio->dxfer_len);
1013                 icl_pdu_free(response);
1014                 iscsi_session_reconnect(is);
1015                 return;
1016         }
1017
1018         icl_pdu_get_data(response, 0, csio->data_ptr + io->io_received, data_segment_len);
1019         io->io_received += data_segment_len;
1020
1021         /*
1022          * XXX: Check DataSN.
1023          * XXX: Check F.
1024          */
1025         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1026                 /*
1027                  * Nothing more to do.
1028                  */
1029                 icl_pdu_free(response);
1030                 return;
1031         }
1032
1033         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1034         if (bhsdi->bhsdi_status == 0) {
1035                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
1036         } else {
1037                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1038                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
1039                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1040                 }
1041                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1042                 csio->scsi_status = bhsdi->bhsdi_status;
1043         }
1044
1045         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1046                 KASSERT(io->io_received <= csio->dxfer_len,
1047                     ("io->io_received > csio->dxfer_len"));
1048                 if (io->io_received < csio->dxfer_len) {
1049                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1050                         if (csio->resid != csio->dxfer_len - io->io_received) {
1051                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1052                                     "target indicates %d, we calculated %zd",
1053                                     csio->resid,
1054                                     csio->dxfer_len - io->io_received);
1055                         }
1056                         csio->resid = csio->dxfer_len - io->io_received;
1057                 }
1058         }
1059
1060         xpt_done(io->io_ccb);
1061         iscsi_outstanding_remove(is, io);
1062         icl_pdu_free(response);
1063 }
1064
1065 static void
1066 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1067 {
1068
1069         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1070         icl_pdu_free(response);
1071 }
1072
1073 static void
1074 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1075 {
1076         struct icl_pdu *request;
1077         struct iscsi_session *is;
1078         struct iscsi_bhs_r2t *bhsr2t;
1079         struct iscsi_bhs_data_out *bhsdo;
1080         struct iscsi_outstanding *io;
1081         struct ccb_scsiio *csio;
1082         size_t off, len, total_len;
1083         int error;
1084
1085         is = PDU_SESSION(response);
1086
1087         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1088         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1089         if (io == NULL || io->io_ccb == NULL) {
1090                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1091                     bhsr2t->bhsr2t_initiator_task_tag);
1092                 icl_pdu_free(response);
1093                 iscsi_session_reconnect(is);
1094                 return;
1095         }
1096
1097         csio = &io->io_ccb->csio;
1098
1099         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1100                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1101                 icl_pdu_free(response);
1102                 iscsi_session_reconnect(is);
1103                 return;
1104         }
1105
1106         /*
1107          * XXX: Verify R2TSN.
1108          */
1109
1110         io->io_datasn = 0;
1111
1112         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1113         if (off > csio->dxfer_len) {
1114                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1115                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1116                 icl_pdu_free(response);
1117                 iscsi_session_reconnect(is);
1118                 return;
1119         }
1120
1121         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1122         if (total_len == 0 || total_len > csio->dxfer_len) {
1123                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1124                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1125                 icl_pdu_free(response);
1126                 iscsi_session_reconnect(is);
1127                 return;
1128         }
1129
1130         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1131
1132         for (;;) {
1133                 len = total_len;
1134
1135                 if (len > is->is_max_data_segment_length)
1136                         len = is->is_max_data_segment_length;
1137
1138                 if (off + len > csio->dxfer_len) {
1139                         ISCSI_SESSION_WARN(is, "target requested invalid "
1140                             "length/offset %zd, buffer is %d; reconnecting",
1141                             off + len, csio->dxfer_len);
1142                         icl_pdu_free(response);
1143                         iscsi_session_reconnect(is);
1144                         return;
1145                 }
1146
1147                 request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
1148                 if (request == NULL) {
1149                         icl_pdu_free(response);
1150                         iscsi_session_reconnect(is);
1151                         return;
1152                 }
1153
1154                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1155                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1156                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1157                 bhsdo->bhsdo_initiator_task_tag =
1158                     bhsr2t->bhsr2t_initiator_task_tag;
1159                 bhsdo->bhsdo_target_transfer_tag =
1160                     bhsr2t->bhsr2t_target_transfer_tag;
1161                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1162                 bhsdo->bhsdo_buffer_offset = htonl(off);
1163                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1164                     M_NOWAIT);
1165                 if (error != 0) {
1166                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1167                             "reconnecting");
1168                         icl_pdu_free(request);
1169                         icl_pdu_free(response);
1170                         iscsi_session_reconnect(is);
1171                         return;
1172                 }
1173
1174                 off += len;
1175                 total_len -= len;
1176
1177                 if (total_len == 0) {
1178                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1179                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1180                 } else {
1181                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1182                 }
1183
1184                 iscsi_pdu_queue_locked(request);
1185
1186                 if (total_len == 0)
1187                         break;
1188         }
1189
1190         icl_pdu_free(response);
1191 }
1192
1193 static void
1194 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1195 {
1196         struct iscsi_bhs_asynchronous_message *bhsam;
1197         struct iscsi_session *is;
1198
1199         is = PDU_SESSION(response);
1200         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1201         switch (bhsam->bhsam_async_event) {
1202         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1203                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1204                 iscsi_session_logout(is);
1205                 iscsi_session_terminate(is);
1206                 break;
1207         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1208                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1209                 break;
1210         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1211                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1212                 break;
1213         default:
1214                 /*
1215                  * XXX: Technically, we're obligated to also handle
1216                  *      parameter renegotiation.
1217                  */
1218                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1219                 break;
1220         }
1221
1222         icl_pdu_free(response);
1223 }
1224
1225 static void
1226 iscsi_pdu_handle_reject(struct icl_pdu *response)
1227 {
1228         struct iscsi_bhs_reject *bhsr;
1229         struct iscsi_session *is;
1230
1231         is = PDU_SESSION(response);
1232         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1233         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1234             bhsr->bhsr_reason);
1235
1236         icl_pdu_free(response);
1237 }
1238
1239 static int
1240 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1241     struct iscsi_daemon_request *request)
1242 {
1243         struct iscsi_session *is;
1244         int error;
1245
1246         sx_slock(&sc->sc_lock);
1247         for (;;) {
1248                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1249                         ISCSI_SESSION_LOCK(is);
1250                         if (is->is_waiting_for_iscsid)
1251                                 break;
1252                         ISCSI_SESSION_UNLOCK(is);
1253                 }
1254
1255                 if (is == NULL) {
1256                         /*
1257                          * No session requires attention from iscsid(8); wait.
1258                          */
1259                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1260                         if (error != 0) {
1261                                 sx_sunlock(&sc->sc_lock);
1262                                 return (error);
1263                         }
1264                         continue;
1265                 }
1266
1267                 is->is_waiting_for_iscsid = false;
1268                 is->is_login_phase = true;
1269                 is->is_reason[0] = '\0';
1270                 ISCSI_SESSION_UNLOCK(is);
1271
1272                 request->idr_session_id = is->is_id;
1273                 memcpy(&request->idr_isid, &is->is_isid,
1274                     sizeof(request->idr_isid));
1275                 request->idr_tsih = 0;  /* New or reinstated session. */
1276                 memcpy(&request->idr_conf, &is->is_conf,
1277                     sizeof(request->idr_conf));
1278
1279                 sx_sunlock(&sc->sc_lock);
1280                 return (0);
1281         }
1282 }
1283
1284 static int
1285 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1286     struct iscsi_daemon_handoff *handoff)
1287 {
1288         struct iscsi_session *is;
1289         int error;
1290
1291         sx_slock(&sc->sc_lock);
1292
1293         /*
1294          * Find the session to hand off socket to.
1295          */
1296         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1297                 if (is->is_id == handoff->idh_session_id)
1298                         break;
1299         }
1300         if (is == NULL) {
1301                 sx_sunlock(&sc->sc_lock);
1302                 return (ESRCH);
1303         }
1304         ISCSI_SESSION_LOCK(is);
1305         if (is->is_conf.isc_discovery || is->is_terminating) {
1306                 ISCSI_SESSION_UNLOCK(is);
1307                 sx_sunlock(&sc->sc_lock);
1308                 return (EINVAL);
1309         }
1310         if (is->is_connected) {
1311                 /*
1312                  * This might have happened because another iscsid(8)
1313                  * instance handed off the connection in the meantime.
1314                  * Just return.
1315                  */
1316                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1317                     "session");
1318                 ISCSI_SESSION_UNLOCK(is);
1319                 sx_sunlock(&sc->sc_lock);
1320                 return (EBUSY);
1321         }
1322
1323         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1324             sizeof(is->is_target_alias));
1325         is->is_tsih = handoff->idh_tsih;
1326         is->is_statsn = handoff->idh_statsn;
1327         is->is_initial_r2t = handoff->idh_initial_r2t;
1328         is->is_immediate_data = handoff->idh_immediate_data;
1329         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1330         is->is_max_burst_length = handoff->idh_max_burst_length;
1331         is->is_first_burst_length = handoff->idh_first_burst_length;
1332
1333         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1334                 is->is_conn->ic_header_crc32c = true;
1335         else
1336                 is->is_conn->ic_header_crc32c = false;
1337         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1338                 is->is_conn->ic_data_crc32c = true;
1339         else
1340                 is->is_conn->ic_data_crc32c = false;
1341
1342         is->is_cmdsn = 0;
1343         is->is_expcmdsn = 0;
1344         is->is_maxcmdsn = 0;
1345         is->is_waiting_for_iscsid = false;
1346         is->is_login_phase = false;
1347         is->is_timeout = 0;
1348         is->is_connected = true;
1349         is->is_reason[0] = '\0';
1350
1351         ISCSI_SESSION_UNLOCK(is);
1352
1353 #ifdef ICL_KERNEL_PROXY
1354         if (handoff->idh_socket != 0) {
1355 #endif
1356                 /*
1357                  * Handoff without using ICL proxy.
1358                  */
1359                 error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1360                 if (error != 0) {
1361                         sx_sunlock(&sc->sc_lock);
1362                         iscsi_session_terminate(is);
1363                         return (error);
1364                 }
1365 #ifdef ICL_KERNEL_PROXY
1366         }
1367 #endif
1368
1369         sx_sunlock(&sc->sc_lock);
1370
1371         if (is->is_sim != NULL) {
1372                 /*
1373                  * When reconnecting, there already is SIM allocated for the session.
1374                  */
1375                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1376                 ISCSI_SESSION_LOCK(is);
1377                 ISCSI_SESSION_DEBUG(is, "releasing");
1378                 xpt_release_simq(is->is_sim, 1);
1379                 is->is_simq_frozen = false;
1380                 ISCSI_SESSION_UNLOCK(is);
1381
1382         } else {
1383                 ISCSI_SESSION_LOCK(is);
1384                 is->is_devq = cam_simq_alloc(maxtags);
1385                 if (is->is_devq == NULL) {
1386                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1387                         iscsi_session_terminate(is);
1388                         return (ENOMEM);
1389                 }
1390
1391                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1392                     is, is->is_id /* unit */, &is->is_lock,
1393                     1, maxtags, is->is_devq);
1394                 if (is->is_sim == NULL) {
1395                         ISCSI_SESSION_UNLOCK(is);
1396                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1397                         cam_simq_free(is->is_devq);
1398                         iscsi_session_terminate(is);
1399                         return (ENOMEM);
1400                 }
1401
1402                 error = xpt_bus_register(is->is_sim, NULL, 0);
1403                 if (error != 0) {
1404                         ISCSI_SESSION_UNLOCK(is);
1405                         ISCSI_SESSION_WARN(is, "failed to register bus");
1406                         iscsi_session_terminate(is);
1407                         return (ENOMEM);
1408                 }
1409
1410                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1411                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1412                     CAM_LUN_WILDCARD);
1413                 if (error != CAM_REQ_CMP) {
1414                         ISCSI_SESSION_UNLOCK(is);
1415                         ISCSI_SESSION_WARN(is, "failed to create path");
1416                         iscsi_session_terminate(is);
1417                         return (ENOMEM);
1418                 }
1419                 ISCSI_SESSION_UNLOCK(is);
1420         }
1421
1422         return (0);
1423 }
1424
1425 static int
1426 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1427     struct iscsi_daemon_fail *fail)
1428 {
1429         struct iscsi_session *is;
1430
1431         sx_slock(&sc->sc_lock);
1432
1433         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1434                 if (is->is_id == fail->idf_session_id)
1435                         break;
1436         }
1437         if (is == NULL) {
1438                 sx_sunlock(&sc->sc_lock);
1439                 return (ESRCH);
1440         }
1441         ISCSI_SESSION_LOCK(is);
1442         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1443             fail->idf_reason);
1444         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1445         //is->is_waiting_for_iscsid = false;
1446         //is->is_login_phase = true;
1447         //iscsi_session_reconnect(is);
1448         ISCSI_SESSION_UNLOCK(is);
1449         sx_sunlock(&sc->sc_lock);
1450
1451         return (0);
1452 }
1453
1454 #ifdef ICL_KERNEL_PROXY
1455 static int
1456 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1457     struct iscsi_daemon_connect *idc)
1458 {
1459         struct iscsi_session *is;
1460         struct sockaddr *from_sa, *to_sa;
1461         int error;
1462
1463         sx_slock(&sc->sc_lock);
1464         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1465                 if (is->is_id == idc->idc_session_id)
1466                         break;
1467         }
1468         if (is == NULL) {
1469                 sx_sunlock(&sc->sc_lock);
1470                 return (ESRCH);
1471         }
1472         sx_sunlock(&sc->sc_lock);
1473
1474         if (idc->idc_from_addrlen > 0) {
1475                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1476                 if (error != 0) {
1477                         ISCSI_SESSION_WARN(is,
1478                             "getsockaddr failed with error %d", error);
1479                         return (error);
1480                 }
1481         } else {
1482                 from_sa = NULL;
1483         }
1484         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1485         if (error != 0) {
1486                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1487                     error);
1488                 free(from_sa, M_SONAME);
1489                 return (error);
1490         }
1491
1492         ISCSI_SESSION_LOCK(is);
1493         is->is_waiting_for_iscsid = false;
1494         is->is_login_phase = true;
1495         is->is_timeout = 0;
1496         ISCSI_SESSION_UNLOCK(is);
1497
1498         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1499             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1500         free(from_sa, M_SONAME);
1501         free(to_sa, M_SONAME);
1502
1503         /*
1504          * Digests are always disabled during login phase.
1505          */
1506         is->is_conn->ic_header_crc32c = false;
1507         is->is_conn->ic_data_crc32c = false;
1508
1509         return (error);
1510 }
1511
1512 static int
1513 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1514     struct iscsi_daemon_send *ids)
1515 {
1516         struct iscsi_session *is;
1517         struct icl_pdu *ip;
1518         size_t datalen;
1519         void *data;
1520         int error;
1521
1522         sx_slock(&sc->sc_lock);
1523         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1524                 if (is->is_id == ids->ids_session_id)
1525                         break;
1526         }
1527         if (is == NULL) {
1528                 sx_sunlock(&sc->sc_lock);
1529                 return (ESRCH);
1530         }
1531         sx_sunlock(&sc->sc_lock);
1532
1533         if (is->is_login_phase == false)
1534                 return (EBUSY);
1535
1536         if (is->is_terminating || is->is_reconnecting)
1537                 return (EIO);
1538
1539         datalen = ids->ids_data_segment_len;
1540         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1541                 return (EINVAL);
1542         if (datalen > 0) {
1543                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1544                 error = copyin(ids->ids_data_segment, data, datalen);
1545                 if (error != 0) {
1546                         free(data, M_ISCSI);
1547                         return (error);
1548                 }
1549         }
1550
1551         ip = icl_pdu_new_bhs(is->is_conn, M_WAITOK);
1552         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1553         if (datalen > 0) {
1554                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1555                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1556                 free(data, M_ISCSI);
1557         }
1558         icl_pdu_queue(ip);
1559
1560         return (0);
1561 }
1562
1563 static int
1564 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1565     struct iscsi_daemon_receive *idr)
1566 {
1567         struct iscsi_session *is;
1568         struct icl_pdu *ip;
1569         void *data;
1570
1571         sx_slock(&sc->sc_lock);
1572         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1573                 if (is->is_id == idr->idr_session_id)
1574                         break;
1575         }
1576         if (is == NULL) {
1577                 sx_sunlock(&sc->sc_lock);
1578                 return (ESRCH);
1579         }
1580         sx_sunlock(&sc->sc_lock);
1581
1582         if (is->is_login_phase == false)
1583                 return (EBUSY);
1584
1585         ISCSI_SESSION_LOCK(is);
1586         while (is->is_login_pdu == NULL &&
1587             is->is_terminating == false &&
1588             is->is_reconnecting == false)
1589                 cv_wait(&is->is_login_cv, &is->is_lock);
1590         if (is->is_terminating || is->is_reconnecting) {
1591                 ISCSI_SESSION_UNLOCK(is);
1592                 return (EIO);
1593         }
1594         ip = is->is_login_pdu;
1595         is->is_login_pdu = NULL;
1596         ISCSI_SESSION_UNLOCK(is);
1597
1598         if (ip->ip_data_len > idr->idr_data_segment_len) {
1599                 icl_pdu_free(ip);
1600                 return (EMSGSIZE);
1601         }
1602
1603         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1604         if (ip->ip_data_len > 0) {
1605                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1606                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1607                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1608                 free(data, M_ISCSI);
1609         }
1610
1611         icl_pdu_free(ip);
1612
1613         return (0);
1614 }
1615 #endif /* ICL_KERNEL_PROXY */
1616
1617 static void
1618 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1619 {
1620         /*
1621          * Just make sure all the fields are null-terminated.
1622          *
1623          * XXX: This is not particularly secure.  We should
1624          *      create our own conf and then copy in relevant
1625          *      fields.
1626          */
1627         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1628         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1629         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1630         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1631         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1632         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1633         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1634         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1635         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1636 }
1637
1638 static bool
1639 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1640 {
1641
1642         if (isc->isc_initiator[0] == '\0') {
1643                 ISCSI_DEBUG("empty isc_initiator");
1644                 return (false);
1645         }
1646
1647         if (isc->isc_target_addr[0] == '\0') {
1648                 ISCSI_DEBUG("empty isc_target_addr");
1649                 return (false);
1650         }
1651
1652         if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1653                 ISCSI_DEBUG("non-empty isc_target for discovery session");
1654                 return (false);
1655         }
1656
1657         if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1658                 ISCSI_DEBUG("empty isc_target for non-discovery session");
1659                 return (false);
1660         }
1661
1662         return (true);
1663 }
1664
1665 static int
1666 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1667 {
1668         struct iscsi_session *is;
1669         const struct iscsi_session *is2;
1670         int error;
1671
1672         iscsi_sanitize_session_conf(&isa->isa_conf);
1673         if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1674                 return (EINVAL);
1675
1676         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1677         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1678
1679         sx_xlock(&sc->sc_lock);
1680
1681         /*
1682          * Prevent duplicates.
1683          */
1684         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1685                 if (!!is->is_conf.isc_discovery !=
1686                     !!is2->is_conf.isc_discovery)
1687                         continue;
1688
1689                 if (strcmp(is->is_conf.isc_target_addr,
1690                     is2->is_conf.isc_target_addr) != 0)
1691                         continue;
1692
1693                 if (is->is_conf.isc_discovery == 0 &&
1694                     strcmp(is->is_conf.isc_target,
1695                     is2->is_conf.isc_target) != 0)
1696                         continue;
1697
1698                 sx_xunlock(&sc->sc_lock);
1699                 free(is, M_ISCSI);
1700                 return (EBUSY);
1701         }
1702
1703         is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1704         is->is_conn->ic_receive = iscsi_receive_callback;
1705         is->is_conn->ic_error = iscsi_error_callback;
1706         is->is_conn->ic_prv0 = is;
1707         TAILQ_INIT(&is->is_outstanding);
1708         STAILQ_INIT(&is->is_postponed);
1709         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1710         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1711 #ifdef ICL_KERNEL_PROXY
1712         cv_init(&is->is_login_cv, "iscsi_login");
1713 #endif
1714
1715         is->is_softc = sc;
1716         sc->sc_last_session_id++;
1717         is->is_id = sc->sc_last_session_id;
1718         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1719         arc4rand(&is->is_isid[1], 5, 0);
1720         is->is_tsih = 0;
1721         callout_init(&is->is_callout, 1);
1722         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1723         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1724
1725         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1726         if (error != 0) {
1727                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1728                 return (error);
1729         }
1730
1731         /*
1732          * Trigger immediate reconnection.
1733          */
1734         ISCSI_SESSION_LOCK(is);
1735         is->is_waiting_for_iscsid = true;
1736         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1737         ISCSI_SESSION_UNLOCK(is);
1738         cv_signal(&sc->sc_cv);
1739
1740         sx_xunlock(&sc->sc_lock);
1741
1742         return (0);
1743 }
1744
1745 static bool
1746 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1747     unsigned int id2, const struct iscsi_session_conf *c2)
1748 {
1749         if (id2 == 0 && c2->isc_target[0] == '\0' &&
1750             c2->isc_target_addr[0] == '\0')
1751                 return (true);
1752         if (id2 != 0 && id2 == id1)
1753                 return (true);
1754         if (c2->isc_target[0] != '\0' &&
1755             strcmp(c1->isc_target, c2->isc_target) == 0)
1756                 return (true);
1757         if (c2->isc_target_addr[0] != '\0' &&
1758             strcmp(c1->isc_target_addr, c2->isc_target_addr) == 0)
1759                 return (true);
1760         return (false);
1761 }
1762
1763 static int
1764 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1765     struct iscsi_session_remove *isr)
1766 {
1767         struct iscsi_session *is, *tmp;
1768         bool found = false;
1769
1770         iscsi_sanitize_session_conf(&isr->isr_conf);
1771
1772         sx_xlock(&sc->sc_lock);
1773         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1774                 ISCSI_SESSION_LOCK(is);
1775                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1776                     isr->isr_session_id, &isr->isr_conf)) {
1777                         found = true;
1778                         iscsi_session_logout(is);
1779                         iscsi_session_terminate(is);
1780                 }
1781                 ISCSI_SESSION_UNLOCK(is);
1782         }
1783         sx_xunlock(&sc->sc_lock);
1784
1785         if (!found)
1786                 return (ESRCH);
1787
1788         return (0);
1789 }
1790
1791 static int
1792 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1793 {
1794         int error;
1795         unsigned int i = 0;
1796         struct iscsi_session *is;
1797         struct iscsi_session_state iss;
1798
1799         sx_slock(&sc->sc_lock);
1800         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1801                 if (i >= isl->isl_nentries) {
1802                         sx_sunlock(&sc->sc_lock);
1803                         return (EMSGSIZE);
1804                 }
1805                 memset(&iss, 0, sizeof(iss));
1806                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1807                 iss.iss_id = is->is_id;
1808                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1809                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1810
1811                 if (is->is_conn->ic_header_crc32c)
1812                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1813                 else
1814                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1815
1816                 if (is->is_conn->ic_data_crc32c)
1817                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1818                 else
1819                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1820
1821                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1822                 iss.iss_immediate_data = is->is_immediate_data;
1823                 iss.iss_connected = is->is_connected;
1824
1825                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1826                 if (error != 0) {
1827                         sx_sunlock(&sc->sc_lock);
1828                         return (error);
1829                 }
1830                 i++;
1831         }
1832         sx_sunlock(&sc->sc_lock);
1833
1834         isl->isl_nentries = i;
1835
1836         return (0);
1837 }
1838
1839 static int
1840 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
1841     struct iscsi_session_modify *ism)
1842 {
1843         struct iscsi_session *is;
1844
1845         iscsi_sanitize_session_conf(&ism->ism_conf);
1846         if (iscsi_valid_session_conf(&ism->ism_conf) == false)
1847                 return (EINVAL);
1848
1849         sx_xlock(&sc->sc_lock);
1850         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1851                 ISCSI_SESSION_LOCK(is);
1852                 if (is->is_id == ism->ism_session_id)
1853                         break;
1854                 ISCSI_SESSION_UNLOCK(is);
1855         }
1856         if (is == NULL) {
1857                 sx_xunlock(&sc->sc_lock);
1858                 return (ESRCH);
1859         }
1860         sx_xunlock(&sc->sc_lock);
1861
1862         memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
1863         ISCSI_SESSION_UNLOCK(is);
1864
1865         iscsi_session_reconnect(is);
1866
1867         return (0);
1868 }
1869
1870 static int
1871 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1872     struct thread *td)
1873 {
1874         struct iscsi_softc *sc;
1875
1876         sc = dev->si_drv1;
1877
1878         switch (cmd) {
1879         case ISCSIDWAIT:
1880                 return (iscsi_ioctl_daemon_wait(sc,
1881                     (struct iscsi_daemon_request *)arg));
1882         case ISCSIDHANDOFF:
1883                 return (iscsi_ioctl_daemon_handoff(sc,
1884                     (struct iscsi_daemon_handoff *)arg));
1885         case ISCSIDFAIL:
1886                 return (iscsi_ioctl_daemon_fail(sc,
1887                     (struct iscsi_daemon_fail *)arg));
1888 #ifdef ICL_KERNEL_PROXY
1889         case ISCSIDCONNECT:
1890                 return (iscsi_ioctl_daemon_connect(sc,
1891                     (struct iscsi_daemon_connect *)arg));
1892         case ISCSIDSEND:
1893                 return (iscsi_ioctl_daemon_send(sc,
1894                     (struct iscsi_daemon_send *)arg));
1895         case ISCSIDRECEIVE:
1896                 return (iscsi_ioctl_daemon_receive(sc,
1897                     (struct iscsi_daemon_receive *)arg));
1898 #endif /* ICL_KERNEL_PROXY */
1899         case ISCSISADD:
1900                 return (iscsi_ioctl_session_add(sc,
1901                     (struct iscsi_session_add *)arg));
1902         case ISCSISREMOVE:
1903                 return (iscsi_ioctl_session_remove(sc,
1904                     (struct iscsi_session_remove *)arg));
1905         case ISCSISLIST:
1906                 return (iscsi_ioctl_session_list(sc,
1907                     (struct iscsi_session_list *)arg));
1908         case ISCSISMODIFY:
1909                 return (iscsi_ioctl_session_modify(sc,
1910                     (struct iscsi_session_modify *)arg));
1911         default:
1912                 return (EINVAL);
1913         }
1914 }
1915
1916 static uint64_t
1917 iscsi_encode_lun(uint32_t lun)
1918 {
1919         uint8_t encoded[8];
1920         uint64_t result;
1921
1922         memset(encoded, 0, sizeof(encoded));
1923
1924         if (lun < 256) {
1925                 /*
1926                  * Peripheral device addressing.
1927                  */
1928                 encoded[1] = lun;
1929         } else if (lun < 16384) {
1930                 /*
1931                  * Flat space addressing.
1932                  */
1933                 encoded[0] = 0x40;
1934                 encoded[0] |= (lun >> 8) & 0x3f;
1935                 encoded[1] = lun & 0xff;
1936         } else {
1937                 /*
1938                  * Extended flat space addressing.
1939                  */
1940                 encoded[0] = 0xd2;
1941                 encoded[1] = lun >> 16;
1942                 encoded[2] = lun >> 8;
1943                 encoded[3] = lun;
1944         }
1945
1946         memcpy(&result, encoded, sizeof(result));
1947         return (result);
1948 }
1949
1950 static struct iscsi_outstanding *
1951 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1952 {
1953         struct iscsi_outstanding *io;
1954
1955         ISCSI_SESSION_LOCK_ASSERT(is);
1956
1957         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1958                 if (io->io_initiator_task_tag == initiator_task_tag)
1959                         return (io);
1960         }
1961         return (NULL);
1962 }
1963
1964 static struct iscsi_outstanding *
1965 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
1966 {
1967         struct iscsi_outstanding *io;
1968
1969         ISCSI_SESSION_LOCK_ASSERT(is);
1970
1971         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1972                 if (io->io_ccb == ccb)
1973                         return (io);
1974         }
1975         return (NULL);
1976 }
1977
1978 static struct iscsi_outstanding *
1979 iscsi_outstanding_add(struct iscsi_session *is,
1980     uint32_t initiator_task_tag, union ccb *ccb)
1981 {
1982         struct iscsi_outstanding *io;
1983
1984         ISCSI_SESSION_LOCK_ASSERT(is);
1985
1986         KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
1987             ("initiator_task_tag 0x%x already added", initiator_task_tag));
1988
1989         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
1990         if (io == NULL) {
1991                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
1992                 return (NULL);
1993         }
1994         io->io_initiator_task_tag = initiator_task_tag;
1995         io->io_ccb = ccb;
1996         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
1997         return (io);
1998 }
1999
2000 static void
2001 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
2002 {
2003
2004         ISCSI_SESSION_LOCK_ASSERT(is);
2005
2006         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
2007         uma_zfree(iscsi_outstanding_zone, io);
2008 }
2009
2010 static void
2011 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
2012 {
2013         struct icl_pdu *request;
2014         struct iscsi_bhs_task_management_request *bhstmr;
2015         struct ccb_abort *cab = &ccb->cab;
2016         struct iscsi_outstanding *io, *aio;
2017
2018         ISCSI_SESSION_LOCK_ASSERT(is);
2019
2020 #if 0
2021         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2022 #else
2023         if (is->is_login_phase) {
2024                 ccb->ccb_h.status = CAM_REQ_ABORTED;
2025                 xpt_done(ccb);
2026                 return;
2027         }
2028 #endif
2029
2030         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
2031         if (aio == NULL) {
2032                 ccb->ccb_h.status = CAM_REQ_CMP;
2033                 xpt_done(ccb);
2034                 return;
2035         }
2036
2037         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
2038         if (request == NULL) {
2039                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2040                 xpt_done(ccb);
2041                 return;
2042         }
2043
2044         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2045         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
2046         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
2047
2048         bhstmr->bhstmr_lun = iscsi_encode_lun(ccb->ccb_h.target_lun);
2049         bhstmr->bhstmr_initiator_task_tag = is->is_initiator_task_tag;
2050         is->is_initiator_task_tag++;
2051         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2052
2053         io = iscsi_outstanding_add(is, bhstmr->bhstmr_initiator_task_tag, NULL);
2054         if (io == NULL) {
2055                 icl_pdu_free(request);
2056                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2057                 xpt_done(ccb);
2058                 return;
2059         }
2060         io->io_datasn = aio->io_initiator_task_tag;
2061         iscsi_pdu_queue_locked(request);
2062 }
2063
2064 static void
2065 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2066 {
2067         struct icl_pdu *request;
2068         struct iscsi_bhs_scsi_command *bhssc;
2069         struct ccb_scsiio *csio;
2070         struct iscsi_outstanding *io;
2071         size_t len;
2072         int error;
2073
2074         ISCSI_SESSION_LOCK_ASSERT(is);
2075
2076 #if 0
2077         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2078 #else
2079         if (is->is_login_phase) {
2080                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2081                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2082                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2083                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2084                 }
2085                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2086                 xpt_done(ccb);
2087                 return;
2088         }
2089 #endif
2090
2091         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
2092         if (request == NULL) {
2093                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2094                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2095                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2096                 }
2097                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2098                 xpt_done(ccb);
2099                 return;
2100         }
2101
2102         csio = &ccb->csio;
2103         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2104         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2105         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2106         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2107         case CAM_DIR_IN:
2108                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2109                 break;
2110         case CAM_DIR_OUT:
2111                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2112                 break;
2113         }
2114
2115         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2116                 switch (csio->tag_action) {
2117                 case MSG_HEAD_OF_Q_TAG:
2118                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2119                         break;
2120                 case MSG_ORDERED_Q_TAG:
2121                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2122                         break;
2123                 case MSG_ACA_TASK:
2124                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2125                         break;
2126                 case MSG_SIMPLE_Q_TAG:
2127                 default:
2128                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2129                         break;
2130                 }
2131         } else
2132                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2133
2134         bhssc->bhssc_lun = iscsi_encode_lun(csio->ccb_h.target_lun);
2135         bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
2136         is->is_initiator_task_tag++;
2137         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2138         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2139             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2140
2141         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2142                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2143         else
2144                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2145
2146         io = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
2147         if (io == NULL) {
2148                 icl_pdu_free(request);
2149                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2150                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2151                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2152                 }
2153                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2154                 xpt_done(ccb);
2155                 return;
2156         }
2157
2158         if (is->is_immediate_data &&
2159             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2160                 len = csio->dxfer_len;
2161                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2162                 if (len > is->is_first_burst_length) {
2163                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
2164                         len = is->is_first_burst_length;
2165                 }
2166
2167                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2168                 if (error != 0) {
2169                         icl_pdu_free(request);
2170                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2171                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2172                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2173                         }
2174                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2175                         xpt_done(ccb);
2176                         return;
2177                 }
2178         }
2179         iscsi_pdu_queue_locked(request);
2180 }
2181
2182 static void
2183 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2184 {
2185         struct iscsi_session *is;
2186
2187         is = cam_sim_softc(sim);
2188
2189         ISCSI_SESSION_LOCK_ASSERT(is);
2190
2191         if (is->is_terminating ||
2192             (is->is_connected == false && fail_on_disconnection)) {
2193                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2194                 xpt_done(ccb);
2195                 return;
2196         }
2197
2198         switch (ccb->ccb_h.func_code) {
2199         case XPT_PATH_INQ:
2200         {
2201                 struct ccb_pathinq *cpi = &ccb->cpi;
2202
2203                 cpi->version_num = 1;
2204                 cpi->hba_inquiry = PI_TAG_ABLE;
2205                 cpi->target_sprt = 0;
2206                 //cpi->hba_misc = PIM_NOBUSRESET;
2207                 cpi->hba_misc = 0;
2208                 cpi->hba_eng_cnt = 0;
2209                 cpi->max_target = 0;
2210                 cpi->max_lun = 255;
2211                 //cpi->initiator_id = 0; /* XXX */
2212                 cpi->initiator_id = 64; /* XXX */
2213                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2214                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2215                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2216                 cpi->unit_number = cam_sim_unit(sim);
2217                 cpi->bus_id = cam_sim_bus(sim);
2218                 cpi->base_transfer_speed = 150000; /* XXX */
2219                 cpi->transport = XPORT_ISCSI;
2220                 cpi->transport_version = 0;
2221                 cpi->protocol = PROTO_SCSI;
2222                 cpi->protocol_version = SCSI_REV_SPC3;
2223                 cpi->maxio = MAXPHYS;
2224                 cpi->ccb_h.status = CAM_REQ_CMP;
2225                 break;
2226         }
2227         case XPT_GET_TRAN_SETTINGS:
2228         {
2229                 struct ccb_trans_settings       *cts;
2230                 struct ccb_trans_settings_scsi  *scsi;
2231
2232                 cts = &ccb->cts;
2233                 scsi = &cts->proto_specific.scsi;
2234
2235                 cts->protocol = PROTO_SCSI;
2236                 cts->protocol_version = SCSI_REV_SPC3;
2237                 cts->transport = XPORT_ISCSI;
2238                 cts->transport_version = 0;
2239                 scsi->valid = CTS_SCSI_VALID_TQ;
2240                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2241                 cts->ccb_h.status = CAM_REQ_CMP;
2242                 break;
2243         }
2244         case XPT_CALC_GEOMETRY:
2245                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2246                 ccb->ccb_h.status = CAM_REQ_CMP;
2247                 break;
2248 #if 0
2249         /*
2250          * XXX: What's the point?
2251          */
2252         case XPT_RESET_BUS:
2253         case XPT_TERM_IO:
2254                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2255                 ccb->ccb_h.status = CAM_REQ_CMP;
2256                 break;
2257 #endif
2258         case XPT_ABORT:
2259                 iscsi_action_abort(is, ccb);
2260                 return;
2261         case XPT_SCSI_IO:
2262                 iscsi_action_scsiio(is, ccb);
2263                 return;
2264         default:
2265 #if 0
2266                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2267 #endif
2268                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2269                 break;
2270         }
2271         xpt_done(ccb);
2272 }
2273
2274 static void
2275 iscsi_poll(struct cam_sim *sim)
2276 {
2277
2278         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2279 }
2280
2281 static void
2282 iscsi_shutdown(struct iscsi_softc *sc)
2283 {
2284         struct iscsi_session *is;
2285
2286         ISCSI_DEBUG("removing all sessions due to shutdown");
2287
2288         sx_slock(&sc->sc_lock);
2289         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2290                 iscsi_session_terminate(is);
2291         sx_sunlock(&sc->sc_lock);
2292 }
2293
2294 static int
2295 iscsi_load(void)
2296 {
2297         int error;
2298
2299         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2300         sx_init(&sc->sc_lock, "iscsi");
2301         TAILQ_INIT(&sc->sc_sessions);
2302         cv_init(&sc->sc_cv, "iscsi_cv");
2303
2304         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2305             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2306             UMA_ALIGN_PTR, 0);
2307
2308         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2309             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2310         if (error != 0) {
2311                 ISCSI_WARN("failed to create device node, error %d", error);
2312                 return (error);
2313         }
2314         sc->sc_cdev->si_drv1 = sc;
2315
2316         /*
2317          * Note that this needs to get run before dashutdown().  Otherwise,
2318          * when rebooting with iSCSI session with outstanding requests,
2319          * but disconnected, dashutdown() will hang on cam_periph_runccb().
2320          */
2321         sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2322             iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2323
2324         return (0);
2325 }
2326
2327 static int
2328 iscsi_unload(void)
2329 {
2330         struct iscsi_session *is, *tmp;
2331
2332         if (sc->sc_cdev != NULL) {
2333                 ISCSI_DEBUG("removing device node");
2334                 destroy_dev(sc->sc_cdev);
2335                 ISCSI_DEBUG("device node removed");
2336         }
2337
2338         if (sc->sc_shutdown_eh != NULL)
2339                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2340
2341         sx_slock(&sc->sc_lock);
2342         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2343                 iscsi_session_terminate(is);
2344         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2345                 ISCSI_DEBUG("waiting for sessions to terminate");
2346                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2347         }
2348         ISCSI_DEBUG("all sessions terminated");
2349         sx_sunlock(&sc->sc_lock);
2350
2351         uma_zdestroy(iscsi_outstanding_zone);
2352         sx_destroy(&sc->sc_lock);
2353         cv_destroy(&sc->sc_cv);
2354         free(sc, M_ISCSI);
2355         return (0);
2356 }
2357
2358 static int
2359 iscsi_quiesce(void)
2360 {
2361         sx_slock(&sc->sc_lock);
2362         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2363                 sx_sunlock(&sc->sc_lock);
2364                 return (EBUSY);
2365         }
2366         sx_sunlock(&sc->sc_lock);
2367         return (0);
2368 }
2369
2370 static int
2371 iscsi_modevent(module_t mod, int what, void *arg)
2372 {
2373         int error;
2374
2375         switch (what) {
2376         case MOD_LOAD:
2377                 error = iscsi_load();
2378                 break;
2379         case MOD_UNLOAD:
2380                 error = iscsi_unload();
2381                 break;
2382         case MOD_QUIESCE:
2383                 error = iscsi_quiesce();
2384                 break;
2385         default:
2386                 error = EINVAL;
2387                 break;
2388         }
2389         return (error);
2390 }
2391
2392 moduledata_t iscsi_data = {
2393         "iscsi",
2394         iscsi_modevent,
2395         0
2396 };
2397
2398 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2399 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2400 MODULE_DEPEND(iscsi, icl, 1, 1, 1);