sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #include <sys/param.h>
  33 #include <sys/condvar.h>
  34 #include <sys/conf.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/file.h>
  37 #include <sys/kernel.h>
  38 #include <sys/kthread.h>
  39 #include <sys/lock.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mutex.h>
  42 #include <sys/module.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/systm.h>
  45 #include <sys/sx.h>
  46 #include <vm/uma.h>
  47
  48 #include <cam/cam.h>
  49 #include <cam/cam_ccb.h>
  50 #include <cam/cam_xpt.h>
  51 #include <cam/cam_debug.h>
  52 #include <cam/cam_sim.h>
  53 #include <cam/cam_xpt_sim.h>
  54 #include <cam/cam_xpt_periph.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/scsi/scsi_all.h>
  57 #include <cam/scsi/scsi_message.h>
  58
  59 #include "iscsi_ioctl.h"
  60 #include "iscsi.h"
  61 #include "icl.h"
  62 #include "iscsi_proto.h"
  63
  64 #ifdef ICL_KERNEL_PROXY
  65 #include <sys/socketvar.h>
  66 #endif
  67
  68 #ifdef ICL_KERNEL_PROXY
  69 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  70 #endif
  71
  72 /*
  73  * XXX: This is global so the iscsi_unload() can access it.
  74  *      Think about how to do this properly.
  75  */
  76 static struct iscsi_softc       *sc;
  77
  78 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  79 static int debug = 1;
  80 TUNABLE_INT("kern.iscsi.debug", &debug);
  81 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  82     &debug, 0, "Enable debug messages");
  83 static int ping_timeout = 5;
  84 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
  85 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  86     0, "Timeout for ping (NOP-Out) requests, in seconds");
  87 static int iscsid_timeout = 60;
  88 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
  89 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  90     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  91 static int login_timeout = 60;
  92 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
  93 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
  94     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  95 static int maxtags = 255;
  96 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
  97 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
  98     0, "Max number of IO requests queued");
  99 static int fail_on_disconnection = 0;
 100 TUNABLE_INT("kern.iscsi.fail_on_disconnection", &fail_on_disconnection);
 101 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
 102     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
 103
 104 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 105 static uma_zone_t iscsi_outstanding_zone;
 106
 107 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 108 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 109
 110 #define ISCSI_DEBUG(X, ...)                                             \
 111         do {                                                            \
 112                 if (debug > 1)                                          \
 113                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 114         } while (0)
 115
 116 #define ISCSI_WARN(X, ...)                                              \
 117         do {                                                            \
 118                 if (debug > 0) {                                        \
 119                         printf("WARNING: %s: " X "\n",                  \
 120                             __func__, ## __VA_ARGS__);                  \
 121                 }                                                       \
 122         } while (0)
 123
 124 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 125         do {                                                            \
 126                 if (debug > 1) {                                        \
 127                         printf("%s: %s (%s): " X "\n",                  \
 128                             __func__, S->is_conf.isc_target_addr,       \
 129                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 130                 }                                                       \
 131         } while (0)
 132
 133 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 134         do {                                                            \
 135                 if (debug > 0) {                                        \
 136                         printf("WARNING: %s (%s): " X "\n",             \
 137                             S->is_conf.isc_target_addr,                 \
 138                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 139                 }                                                       \
 140         } while (0)
 141
 142 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 143 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 144 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 145
 146 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 147                     int mode, struct thread *td);
 148
 149 static struct cdevsw iscsi_cdevsw = {
 150      .d_version = D_VERSION,
 151      .d_ioctl   = iscsi_ioctl,
 152      .d_name    = "iscsi",
 153 };
 154
 155 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 156 static void     iscsi_pdu_queue(struct icl_pdu *request);
 157 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 158 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 159 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 160 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
 161 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 162 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 163 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 164 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 165 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 166 static void     iscsi_session_reconnect(struct iscsi_session *is);
 167 static void     iscsi_session_terminate(struct iscsi_session *is);
 168 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 169 static void     iscsi_poll(struct cam_sim *sim);
 170 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 171                     uint32_t initiator_task_tag);
 172 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
 173                     uint32_t initiator_task_tag, union ccb *ccb);
 174 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 175                     struct iscsi_outstanding *io);
 176
 177 static bool
 178 iscsi_pdu_prepare(struct icl_pdu *request)
 179 {
 180         struct iscsi_session *is;
 181         struct iscsi_bhs_scsi_command *bhssc;
 182
 183         is = PDU_SESSION(request);
 184
 185         ISCSI_SESSION_LOCK_ASSERT(is);
 186
 187         /*
 188          * We're only using fields common for all the request
 189          * (initiator -> target) PDUs.
 190          */
 191         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 192
 193         /*
 194          * Data-Out PDU does not contain CmdSN.
 195          */
 196         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 197                 if (is->is_cmdsn > is->is_maxcmdsn &&
 198                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 199                         /*
 200                          * Current MaxCmdSN prevents us from sending any more
 201                          * SCSI Command PDUs to the target; postpone the PDU.
 202                          * It will get resent by either iscsi_pdu_queue(),
 203                          * or by maintenance thread.
 204                          */
 205 #if 0
 206                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %d, ExpCmdSN %d, MaxCmdSN %d, opcode 0x%x",
 207                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn, bhssc->bhssc_opcode);
 208 #endif
 209                         return (true);
 210                 }
 211                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 212                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 213                         is->is_cmdsn++;
 214         }
 215         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 216
 217         return (false);
 218 }
 219
 220 static void
 221 iscsi_session_send_postponed(struct iscsi_session *is)
 222 {
 223         struct icl_pdu *request;
 224         bool postpone;
 225
 226         ISCSI_SESSION_LOCK_ASSERT(is);
 227
 228         while (!STAILQ_EMPTY(&is->is_postponed)) {
 229                 request = STAILQ_FIRST(&is->is_postponed);
 230                 postpone = iscsi_pdu_prepare(request);
 231                 if (postpone)
 232                         break;
 233                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 234                 icl_pdu_queue(request);
 235         }
 236 }
 237
 238 static void
 239 iscsi_pdu_queue_locked(struct icl_pdu *request)
 240 {
 241         struct iscsi_session *is;
 242         bool postpone;
 243
 244         is = PDU_SESSION(request);
 245         ISCSI_SESSION_LOCK_ASSERT(is);
 246         iscsi_session_send_postponed(is);
 247         postpone = iscsi_pdu_prepare(request);
 248         if (postpone) {
 249                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 250                 return;
 251         }
 252         icl_pdu_queue(request);
 253 }
 254
 255 static void
 256 iscsi_pdu_queue(struct icl_pdu *request)
 257 {
 258         struct iscsi_session *is;
 259
 260         is = PDU_SESSION(request);
 261         ISCSI_SESSION_LOCK(is);
 262         iscsi_pdu_queue_locked(request);
 263         ISCSI_SESSION_UNLOCK(is);
 264 }
 265
 266 static void
 267 iscsi_session_logout(struct iscsi_session *is)
 268 {
 269         struct icl_pdu *request;
 270         struct iscsi_bhs_logout_request *bhslr;
 271
 272         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 273         if (request == NULL)
 274                 return;
 275
 276         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 277         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 278         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 279         iscsi_pdu_queue_locked(request);
 280 }
 281
 282 static void
 283 iscsi_session_terminate_task(struct iscsi_session *is,
 284     struct iscsi_outstanding *io, bool requeue)
 285 {
 286
 287         if (io->io_ccb != NULL) {
 288                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
 289                 if (requeue)
 290                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 291                 else
 292                         io->io_ccb->ccb_h.status |= CAM_REQ_ABORTED;
 293                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 294                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 295                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 296                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 297                 }
 298                 xpt_done(io->io_ccb);
 299         }
 300         iscsi_outstanding_remove(is, io);
 301 }
 302
 303 static void
 304 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 305 {
 306         struct iscsi_outstanding *io, *tmp;
 307
 308         ISCSI_SESSION_LOCK_ASSERT(is);
 309
 310         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 311                 iscsi_session_terminate_task(is, io, requeue);
 312         }
 313 }
 314
 315 static void
 316 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 317 {
 318         struct icl_pdu *pdu;
 319
 320         ISCSI_SESSION_LOCK_ASSERT(is);
 321
 322         /*
 323          * Don't queue any new PDUs.
 324          */
 325         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 326                 ISCSI_SESSION_DEBUG(is, "freezing");
 327                 xpt_freeze_simq(is->is_sim, 1);
 328                 is->is_simq_frozen = true;
 329         }
 330
 331         /*
 332          * Remove postponed PDUs.
 333          */
 334         while (!STAILQ_EMPTY(&is->is_postponed)) {
 335                 pdu = STAILQ_FIRST(&is->is_postponed);
 336                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 337                 icl_pdu_free(pdu);
 338         }
 339
 340         if (destroy_sim == false) {
 341                 /*
 342                  * Terminate SCSI tasks, asking CAM to requeue them.
 343                  */
 344                 iscsi_session_terminate_tasks(is, true);
 345                 return;
 346         }
 347
 348         iscsi_session_terminate_tasks(is, false);
 349
 350         if (is->is_sim == NULL)
 351                 return;
 352
 353         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 354         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 355
 356         if (is->is_simq_frozen) {
 357                 xpt_release_simq(is->is_sim, 1);
 358                 is->is_simq_frozen = false;
 359         }
 360
 361         xpt_free_path(is->is_path);
 362         is->is_path = NULL;
 363         xpt_bus_deregister(cam_sim_path(is->is_sim));
 364         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 365         is->is_sim = NULL;
 366         is->is_devq = NULL;
 367 }
 368
 369 static void
 370 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 371 {
 372
 373         icl_conn_shutdown(is->is_conn);
 374         icl_conn_close(is->is_conn);
 375
 376         ISCSI_SESSION_LOCK(is);
 377
 378         is->is_connected = false;
 379         is->is_reconnecting = false;
 380         is->is_login_phase = false;
 381
 382 #ifdef ICL_KERNEL_PROXY
 383         if (is->is_login_pdu != NULL) {
 384                 icl_pdu_free(is->is_login_pdu);
 385                 is->is_login_pdu = NULL;
 386         }
 387         cv_signal(&is->is_login_cv);
 388 #endif
 389
 390         if (fail_on_disconnection) {
 391                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
 392                 iscsi_session_cleanup(is, true);
 393         } else {
 394                 iscsi_session_cleanup(is, false);
 395         }
 396
 397         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 398             ("destroying session with active tasks"));
 399         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 400             ("destroying session with postponed PDUs"));
 401
 402         /*
 403          * Request immediate reconnection from iscsid(8).
 404          */
 405         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 406         is->is_waiting_for_iscsid = true;
 407         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 408         is->is_timeout = 0;
 409         ISCSI_SESSION_UNLOCK(is);
 410         cv_signal(&is->is_softc->sc_cv);
 411 }
 412
 413 static void
 414 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 415 {
 416         struct iscsi_softc *sc;
 417
 418         sc = is->is_softc;
 419         sx_xlock(&sc->sc_lock);
 420         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 421         sx_xunlock(&sc->sc_lock);
 422
 423         icl_conn_close(is->is_conn);
 424
 425         ISCSI_SESSION_LOCK(is);
 426
 427         KASSERT(is->is_terminating, ("is_terminating == false"));
 428
 429 #ifdef ICL_KERNEL_PROXY
 430         if (is->is_login_pdu != NULL) {
 431                 icl_pdu_free(is->is_login_pdu);
 432                 is->is_login_pdu = NULL;
 433         }
 434         cv_signal(&is->is_login_cv);
 435 #endif
 436
 437         callout_drain(&is->is_callout);
 438
 439         iscsi_session_cleanup(is, true);
 440
 441         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 442             ("destroying session with active tasks"));
 443         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 444             ("destroying session with postponed PDUs"));
 445
 446         ISCSI_SESSION_UNLOCK(is);
 447
 448         icl_conn_free(is->is_conn);
 449         mtx_destroy(&is->is_lock);
 450         cv_destroy(&is->is_maintenance_cv);
 451 #ifdef ICL_KERNEL_PROXY
 452         cv_destroy(&is->is_login_cv);
 453 #endif
 454         ISCSI_SESSION_DEBUG(is, "terminated");
 455         free(is, M_ISCSI);
 456
 457         /*
 458          * The iscsi_unload() routine might be waiting.
 459          */
 460         cv_signal(&sc->sc_cv);
 461 }
 462
 463 static void
 464 iscsi_maintenance_thread(void *arg)
 465 {
 466         struct iscsi_session *is;
 467
 468         is = arg;
 469
 470         for (;;) {
 471                 ISCSI_SESSION_LOCK(is);
 472                 if (is->is_reconnecting == false &&
 473                     is->is_terminating == false &&
 474                     STAILQ_EMPTY(&is->is_postponed))
 475                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 476
 477                 if (is->is_reconnecting) {
 478                         ISCSI_SESSION_UNLOCK(is);
 479                         iscsi_maintenance_thread_reconnect(is);
 480                         continue;
 481                 }
 482
 483                 if (is->is_terminating) {
 484                         ISCSI_SESSION_UNLOCK(is);
 485                         iscsi_maintenance_thread_terminate(is);
 486                         kthread_exit();
 487                         return;
 488                 }
 489
 490                 iscsi_session_send_postponed(is);
 491                 ISCSI_SESSION_UNLOCK(is);
 492         }
 493 }
 494
 495 static void
 496 iscsi_session_reconnect(struct iscsi_session *is)
 497 {
 498
 499         /*
 500          * XXX: We can't use locking here, because
 501          *      it's being called from various contexts.
 502          *      Hope it doesn't break anything.
 503          */
 504         if (is->is_reconnecting)
 505                 return;
 506
 507         is->is_reconnecting = true;
 508         cv_signal(&is->is_maintenance_cv);
 509 }
 510
 511 static void
 512 iscsi_session_terminate(struct iscsi_session *is)
 513 {
 514         if (is->is_terminating)
 515                 return;
 516
 517         is->is_terminating = true;
 518
 519 #if 0
 520         iscsi_session_logout(is);
 521 #endif
 522         cv_signal(&is->is_maintenance_cv);
 523 }
 524
 525 static void
 526 iscsi_callout(void *context)
 527 {
 528         struct icl_pdu *request;
 529         struct iscsi_bhs_nop_out *bhsno;
 530         struct iscsi_session *is;
 531         bool reconnect_needed = false;
 532
 533         is = context;
 534
 535         if (is->is_terminating)
 536                 return;
 537
 538         callout_schedule(&is->is_callout, 1 * hz);
 539
 540         ISCSI_SESSION_LOCK(is);
 541         is->is_timeout++;
 542
 543         if (is->is_waiting_for_iscsid) {
 544                 if (is->is_timeout > iscsid_timeout) {
 545                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 546                             "for %d seconds; reconnecting",
 547                             is->is_timeout);
 548                         reconnect_needed = true;
 549                 }
 550                 goto out;
 551         }
 552
 553         if (is->is_login_phase) {
 554                 if (is->is_timeout > login_timeout) {
 555                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 556                             "reconnecting", is->is_timeout);
 557                         reconnect_needed = true;
 558                 }
 559                 goto out;
 560         }
 561
 562         if (is->is_timeout >= ping_timeout) {
 563                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 564                     "reconnecting", ping_timeout);
 565                 reconnect_needed = true;
 566                 goto out;
 567         }
 568
 569         ISCSI_SESSION_UNLOCK(is);
 570
 571         /*
 572          * If the ping was reset less than one second ago - which means
 573          * that we've received some PDU during the last second - assume
 574          * the traffic flows correctly and don't bother sending a NOP-Out.
 575          *
 576          * (It's 2 - one for one second, and one for incrementing is_timeout
 577          * earlier in this routine.)
 578          */
 579         if (is->is_timeout < 2)
 580                 return;
 581
 582         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 583         if (request == NULL) {
 584                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 585                 return;
 586         }
 587         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 588         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 589             ISCSI_BHS_OPCODE_IMMEDIATE;
 590         bhsno->bhsno_flags = 0x80;
 591         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 592         iscsi_pdu_queue(request);
 593         return;
 594
 595 out:
 596         ISCSI_SESSION_UNLOCK(is);
 597
 598         if (reconnect_needed)
 599                 iscsi_session_reconnect(is);
 600 }
 601
 602 static void
 603 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 604 {
 605         const struct iscsi_bhs_data_in *bhsdi;
 606         struct iscsi_session *is;
 607         uint32_t expcmdsn, maxcmdsn;
 608
 609         is = PDU_SESSION(response);
 610
 611         ISCSI_SESSION_LOCK_ASSERT(is);
 612
 613         /*
 614          * We're only using fields common for all the response
 615          * (target -> initiator) PDUs.
 616          */
 617         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 618         /*
 619          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 620          * and Residual Count only have meaningful content if the S bit
 621          * is set to 1", so we also need to check the bit specific for
 622          * Data-In PDU.
 623          */
 624         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 625             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 626                 if (ntohl(bhsdi->bhsdi_statsn) < is->is_statsn) {
 627                         ISCSI_SESSION_WARN(is,
 628                             "PDU StatSN %d >= session StatSN %d, opcode 0x%x",
 629                             is->is_statsn, ntohl(bhsdi->bhsdi_statsn),
 630                             bhsdi->bhsdi_opcode);
 631                 }
 632                 is->is_statsn = ntohl(bhsdi->bhsdi_statsn);
 633         }
 634
 635         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 636         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 637
 638         /*
 639          * XXX: Compare using Serial Arithmetic Sense.
 640          */
 641         if (maxcmdsn + 1 < expcmdsn) {
 642                 ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d + 1 < PDU ExpCmdSN %d; ignoring",
 643                     maxcmdsn, expcmdsn);
 644         } else {
 645                 if (maxcmdsn > is->is_maxcmdsn) {
 646                         is->is_maxcmdsn = maxcmdsn;
 647
 648                         /*
 649                          * Command window increased; kick the maintanance thread
 650                          * to send out postponed commands.
 651                          */
 652                         if (!STAILQ_EMPTY(&is->is_postponed))
 653                                 cv_signal(&is->is_maintenance_cv);
 654                 } else if (maxcmdsn < is->is_maxcmdsn) {
 655                         ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d < session MaxCmdSN %d; ignoring",
 656                             maxcmdsn, is->is_maxcmdsn);
 657                 }
 658
 659                 if (expcmdsn > is->is_expcmdsn) {
 660                         is->is_expcmdsn = expcmdsn;
 661                 } else if (expcmdsn < is->is_expcmdsn) {
 662                         ISCSI_SESSION_DEBUG(is, "PDU ExpCmdSN %d < session ExpCmdSN %d; ignoring",
 663                             expcmdsn, is->is_expcmdsn);
 664                 }
 665         }
 666
 667         /*
 668          * Every incoming PDU - not just NOP-In - resets the ping timer.
 669          * The purpose of the timeout is to reset the connection when it stalls;
 670          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 671          * in some queue.
 672          */
 673         is->is_timeout = 0;
 674 }
 675
 676 static void
 677 iscsi_receive_callback(struct icl_pdu *response)
 678 {
 679         struct iscsi_session *is;
 680
 681         is = PDU_SESSION(response);
 682
 683         ISCSI_SESSION_LOCK(is);
 684
 685 #ifdef ICL_KERNEL_PROXY
 686         if (is->is_login_phase) {
 687                 if (is->is_login_pdu == NULL)
 688                         is->is_login_pdu = response;
 689                 else
 690                         icl_pdu_free(response);
 691                 ISCSI_SESSION_UNLOCK(is);
 692                 cv_signal(&is->is_login_cv);
 693                 return;
 694         }
 695 #endif
 696
 697         iscsi_pdu_update_statsn(response);
 698
 699         /*
 700          * The handling routine is responsible for freeing the PDU
 701          * when it's no longer needed.
 702          */
 703         switch (response->ip_bhs->bhs_opcode) {
 704         case ISCSI_BHS_OPCODE_NOP_IN:
 705                 iscsi_pdu_handle_nop_in(response);
 706                 break;
 707         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 708                 iscsi_pdu_handle_scsi_response(response);
 709                 break;
 710         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
 711                 iscsi_pdu_handle_task_response(response);
 712                 break;
 713         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 714                 iscsi_pdu_handle_data_in(response);
 715                 break;
 716         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 717                 iscsi_pdu_handle_logout_response(response);
 718                 break;
 719         case ISCSI_BHS_OPCODE_R2T:
 720                 iscsi_pdu_handle_r2t(response);
 721                 break;
 722         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 723                 iscsi_pdu_handle_async_message(response);
 724                 break;
 725         case ISCSI_BHS_OPCODE_REJECT:
 726                 iscsi_pdu_handle_reject(response);
 727                 break;
 728         default:
 729                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 730                     "opcode 0x%x; reconnecting",
 731                     response->ip_bhs->bhs_opcode);
 732                 iscsi_session_reconnect(is);
 733                 icl_pdu_free(response);
 734         }
 735
 736         ISCSI_SESSION_UNLOCK(is);
 737 }
 738
 739 static void
 740 iscsi_error_callback(struct icl_conn *ic)
 741 {
 742         struct iscsi_session *is;
 743
 744         is = CONN_SESSION(ic);
 745
 746         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 747         iscsi_session_reconnect(is);
 748 }
 749
 750 static void
 751 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 752 {
 753         struct iscsi_session *is;
 754         struct iscsi_bhs_nop_out *bhsno;
 755         struct iscsi_bhs_nop_in *bhsni;
 756         struct icl_pdu *request;
 757         void *data = NULL;
 758         size_t datasize;
 759         int error;
 760
 761         is = PDU_SESSION(response);
 762         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 763
 764         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 765                 /*
 766                  * Nothing to do; iscsi_pdu_update_statsn() already
 767                  * zeroed the timeout.
 768                  */
 769                 icl_pdu_free(response);
 770                 return;
 771         }
 772
 773         datasize = icl_pdu_data_segment_length(response);
 774         if (datasize > 0) {
 775                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 776                 if (data == NULL) {
 777                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 778                             "reconnecting");
 779                         icl_pdu_free(response);
 780                         iscsi_session_reconnect(is);
 781                         return;
 782                 }
 783                 icl_pdu_get_data(response, 0, data, datasize);
 784         }
 785
 786         request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
 787         if (request == NULL) {
 788                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 789                     "reconnecting");
 790                 free(data, M_ISCSI);
 791                 icl_pdu_free(response);
 792                 iscsi_session_reconnect(is);
 793                 return;
 794         }
 795         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 796         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 797             ISCSI_BHS_OPCODE_IMMEDIATE;
 798         bhsno->bhsno_flags = 0x80;
 799         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 800         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 801         if (datasize > 0) {
 802                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 803                 if (error != 0) {
 804                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 805                             "reconnecting");
 806                         free(data, M_ISCSI);
 807                         icl_pdu_free(request);
 808                         icl_pdu_free(response);
 809                         iscsi_session_reconnect(is);
 810                         return;
 811                 }
 812                 free(data, M_ISCSI);
 813         }
 814
 815         icl_pdu_free(response);
 816         iscsi_pdu_queue_locked(request);
 817 }
 818
 819 static void
 820 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 821 {
 822         struct iscsi_bhs_scsi_response *bhssr;
 823         struct iscsi_outstanding *io;
 824         struct iscsi_session *is;
 825         struct ccb_scsiio *csio;
 826         size_t data_segment_len;
 827         uint16_t sense_len;
 828
 829         is = PDU_SESSION(response);
 830
 831         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 832         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 833         if (io == NULL || io->io_ccb == NULL) {
 834                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 835                 icl_pdu_free(response);
 836                 iscsi_session_reconnect(is);
 837                 return;
 838         }
 839
 840         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 841                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 842                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 843                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 844                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 845                 }
 846                 io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 847         } else if (bhssr->bhssr_status == 0) {
 848                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
 849         } else {
 850                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 851                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 852                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 853                 }
 854                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 855                 io->io_ccb->csio.scsi_status = bhssr->bhssr_status;
 856         }
 857
 858         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
 859                 ISCSI_SESSION_WARN(is, "target indicated residual overflow");
 860                 icl_pdu_free(response);
 861                 iscsi_session_reconnect(is);
 862                 return;
 863         }
 864
 865         csio = &io->io_ccb->csio;
 866
 867         data_segment_len = icl_pdu_data_segment_length(response);
 868         if (data_segment_len > 0) {
 869                 if (data_segment_len < sizeof(sense_len)) {
 870                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 871                             data_segment_len);
 872                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 873                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 874                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 875                         }
 876                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 877                         goto out;
 878                 }
 879                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 880                 sense_len = ntohs(sense_len);
 881 #if 0
 882                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 883                     sense_len, data_segment_len);
 884 #endif
 885                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 886                         ISCSI_SESSION_WARN(is, "truncated data segment "
 887                             "(%zd bytes, should be %zd)",
 888                             data_segment_len, sizeof(sense_len) + sense_len);
 889                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 890                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 891                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 892                         }
 893                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 894                         goto out;
 895                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 896                         ISCSI_SESSION_WARN(is, "oversize data segment "
 897                             "(%zd bytes, should be %zd)",
 898                             data_segment_len, sizeof(sense_len) + sense_len);
 899                 if (sense_len > csio->sense_len) {
 900                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 901                             sense_len, csio->sense_len);
 902                         sense_len = csio->sense_len;
 903                 }
 904                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 905                 csio->sense_resid = csio->sense_len - sense_len;
 906                 io->io_ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 907         }
 908
 909 out:
 910         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 911                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 912
 913         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 914                 KASSERT(io->io_received <= csio->dxfer_len,
 915                     ("io->io_received > csio->dxfer_len"));
 916                 if (io->io_received < csio->dxfer_len) {
 917                         if (csio->resid != csio->dxfer_len - io->io_received) {
 918                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 919                                     "target indicates %d, we calculated %zd",
 920                                     csio->resid,
 921                                     csio->dxfer_len - io->io_received);
 922                         }
 923                         csio->resid = csio->dxfer_len - io->io_received;
 924                 }
 925         }
 926
 927         xpt_done(io->io_ccb);
 928         iscsi_outstanding_remove(is, io);
 929         icl_pdu_free(response);
 930 }
 931
 932 static void
 933 iscsi_pdu_handle_task_response(struct icl_pdu *response)
 934 {
 935         struct iscsi_bhs_task_management_response *bhstmr;
 936         struct iscsi_outstanding *io, *aio;
 937         struct iscsi_session *is;
 938
 939         is = PDU_SESSION(response);
 940
 941         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
 942         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
 943         if (io == NULL || io->io_ccb != NULL) {
 944                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
 945                     bhstmr->bhstmr_initiator_task_tag);
 946                 icl_pdu_free(response);
 947                 iscsi_session_reconnect(is);
 948                 return;
 949         }
 950
 951         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
 952                 ISCSI_SESSION_WARN(is, "task response 0x%x",
 953                     bhstmr->bhstmr_response);
 954         } else {
 955                 aio = iscsi_outstanding_find(is, io->io_datasn);
 956                 if (aio != NULL && aio->io_ccb != NULL)
 957                         iscsi_session_terminate_task(is, aio, false);
 958         }
 959
 960         iscsi_outstanding_remove(is, io);
 961         icl_pdu_free(response);
 962 }
 963
 964 static void
 965 iscsi_pdu_handle_data_in(struct icl_pdu *response)
 966 {
 967         struct iscsi_bhs_data_in *bhsdi;
 968         struct iscsi_outstanding *io;
 969         struct iscsi_session *is;
 970         struct ccb_scsiio *csio;
 971         size_t data_segment_len;
 972
 973         is = PDU_SESSION(response);
 974         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
 975         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
 976         if (io == NULL || io->io_ccb == NULL) {
 977                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
 978                 icl_pdu_free(response);
 979                 iscsi_session_reconnect(is);
 980                 return;
 981         }
 982
 983         data_segment_len = icl_pdu_data_segment_length(response);
 984         if (data_segment_len == 0) {
 985                 /*
 986                  * "The sending of 0 length data segments should be avoided,
 987                  * but initiators and targets MUST be able to properly receive
 988                  * 0 length data segments."
 989                  */
 990                 icl_pdu_free(response);
 991                 return;
 992         }
 993
 994         /*
 995          * We need to track this for security reasons - without it, malicious target
 996          * could respond to SCSI READ without sending Data-In PDUs, which would result
 997          * in read operation on the initiator side returning random kernel data.
 998          */
 999         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1000                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1001                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1002                 icl_pdu_free(response);
1003                 iscsi_session_reconnect(is);
1004                 return;
1005         }
1006
1007         csio = &io->io_ccb->csio;
1008
1009         if (io->io_received + data_segment_len > csio->dxfer_len) {
1010                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1011                     "at offset %zd, buffer is %d)",
1012                     data_segment_len, io->io_received, csio->dxfer_len);
1013                 icl_pdu_free(response);
1014                 iscsi_session_reconnect(is);
1015                 return;
1016         }
1017
1018         icl_pdu_get_data(response, 0, csio->data_ptr + io->io_received, data_segment_len);
1019         io->io_received += data_segment_len;
1020
1021         /*
1022          * XXX: Check DataSN.
1023          * XXX: Check F.
1024          */
1025         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1026                 /*
1027                  * Nothing more to do.
1028                  */
1029                 icl_pdu_free(response);
1030                 return;
1031         }
1032
1033         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1034         if (bhsdi->bhsdi_status == 0) {
1035                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
1036         } else {
1037                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1038                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
1039                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1040                 }
1041                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1042                 csio->scsi_status = bhsdi->bhsdi_status;
1043         }
1044
1045         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1046                 KASSERT(io->io_received <= csio->dxfer_len,
1047                     ("io->io_received > csio->dxfer_len"));
1048                 if (io->io_received < csio->dxfer_len) {
1049                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1050                         if (csio->resid != csio->dxfer_len - io->io_received) {
1051                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1052                                     "target indicates %d, we calculated %zd",
1053                                     csio->resid,
1054                                     csio->dxfer_len - io->io_received);
1055                         }
1056                         csio->resid = csio->dxfer_len - io->io_received;
1057                 }
1058         }
1059
1060         xpt_done(io->io_ccb);
1061         iscsi_outstanding_remove(is, io);
1062         icl_pdu_free(response);
1063 }
1064
1065 static void
1066 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1067 {
1068
1069         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1070         icl_pdu_free(response);
1071 }
1072
1073 static void
1074 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1075 {
1076         struct icl_pdu *request;
1077         struct iscsi_session *is;
1078         struct iscsi_bhs_r2t *bhsr2t;
1079         struct iscsi_bhs_data_out *bhsdo;
1080         struct iscsi_outstanding *io;
1081         struct ccb_scsiio *csio;
1082         size_t off, len, total_len;
1083         int error;
1084
1085         is = PDU_SESSION(response);
1086
1087         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1088         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1089         if (io == NULL || io->io_ccb == NULL) {
1090                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1091                     bhsr2t->bhsr2t_initiator_task_tag);
1092                 icl_pdu_free(response);
1093                 iscsi_session_reconnect(is);
1094                 return;
1095         }
1096
1097         csio = &io->io_ccb->csio;
1098
1099         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1100                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1101                 icl_pdu_free(response);
1102                 iscsi_session_reconnect(is);
1103                 return;
1104         }
1105
1106         /*
1107          * XXX: Verify R2TSN.
1108          */
1109
1110         io->io_datasn = 0;
1111
1112         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1113         if (off > csio->dxfer_len) {
1114                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1115                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1116                 icl_pdu_free(response);
1117                 iscsi_session_reconnect(is);
1118                 return;
1119         }
1120
1121         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1122         if (total_len == 0 || total_len > csio->dxfer_len) {
1123                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1124                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1125                 icl_pdu_free(response);
1126                 iscsi_session_reconnect(is);
1127                 return;
1128         }
1129
1130         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1131
1132         for (;;) {
1133                 len = total_len;
1134
1135                 if (len > is->is_max_data_segment_length)
1136                         len = is->is_max_data_segment_length;
1137
1138                 if (off + len > csio->dxfer_len) {
1139                         ISCSI_SESSION_WARN(is, "target requested invalid "
1140                             "length/offset %zd, buffer is %d; reconnecting",
1141                             off + len, csio->dxfer_len);
1142                         icl_pdu_free(response);
1143                         iscsi_session_reconnect(is);
1144                         return;
1145                 }
1146
1147                 request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
1148                 if (request == NULL) {
1149                         icl_pdu_free(response);
1150                         iscsi_session_reconnect(is);
1151                         return;
1152                 }
1153
1154                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1155                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1156                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1157                 bhsdo->bhsdo_initiator_task_tag =
1158                     bhsr2t->bhsr2t_initiator_task_tag;
1159                 bhsdo->bhsdo_target_transfer_tag =
1160                     bhsr2t->bhsr2t_target_transfer_tag;
1161                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1162                 bhsdo->bhsdo_buffer_offset = htonl(off);
1163                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1164                     M_NOWAIT);
1165                 if (error != 0) {
1166                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1167                             "reconnecting");
1168                         icl_pdu_free(request);
1169                         icl_pdu_free(response);
1170                         iscsi_session_reconnect(is);
1171                         return;
1172                 }
1173
1174                 off += len;
1175                 total_len -= len;
1176
1177                 if (total_len == 0) {
1178                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1179                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1180                 } else {
1181                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1182                 }
1183
1184                 iscsi_pdu_queue_locked(request);
1185
1186                 if (total_len == 0)
1187                         break;
1188         }
1189
1190         icl_pdu_free(response);
1191 }
1192
1193 static void
1194 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1195 {
1196         struct iscsi_bhs_asynchronous_message *bhsam;
1197         struct iscsi_session *is;
1198
1199         is = PDU_SESSION(response);
1200         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1201         switch (bhsam->bhsam_async_event) {
1202         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1203                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1204                 iscsi_session_logout(is);
1205                 iscsi_session_terminate(is);
1206                 break;
1207         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1208                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1209                 break;
1210         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1211                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1212                 break;
1213         default:
1214                 /*
1215                  * XXX: Technically, we're obligated to also handle
1216                  *      parameter renegotiation.
1217                  */
1218                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1219                 break;
1220         }
1221
1222         icl_pdu_free(response);
1223 }
1224
1225 static void
1226 iscsi_pdu_handle_reject(struct icl_pdu *response)
1227 {
1228         struct iscsi_bhs_reject *bhsr;
1229         struct iscsi_session *is;
1230
1231         is = PDU_SESSION(response);
1232         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1233         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1234             bhsr->bhsr_reason);
1235
1236         icl_pdu_free(response);
1237 }
1238
1239 static int
1240 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1241     struct iscsi_daemon_request *request)
1242 {
1243         struct iscsi_session *is;
1244         int error;
1245
1246         sx_slock(&sc->sc_lock);
1247         for (;;) {
1248                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1249                         ISCSI_SESSION_LOCK(is);
1250                         if (is->is_waiting_for_iscsid)
1251                                 break;
1252                         ISCSI_SESSION_UNLOCK(is);
1253                 }
1254
1255                 if (is == NULL) {
1256                         /*
1257                          * No session requires attention from iscsid(8); wait.
1258                          */
1259                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1260                         if (error != 0) {
1261                                 sx_sunlock(&sc->sc_lock);
1262                                 return (error);
1263                         }
1264                         continue;
1265                 }
1266
1267                 is->is_waiting_for_iscsid = false;
1268                 is->is_login_phase = true;
1269                 is->is_reason[0] = '\0';
1270                 ISCSI_SESSION_UNLOCK(is);
1271
1272                 request->idr_session_id = is->is_id;
1273                 memcpy(&request->idr_isid, &is->is_isid,
1274                     sizeof(request->idr_isid));
1275                 request->idr_tsih = 0;  /* New or reinstated session. */
1276                 memcpy(&request->idr_conf, &is->is_conf,
1277                     sizeof(request->idr_conf));
1278
1279                 sx_sunlock(&sc->sc_lock);
1280                 return (0);
1281         }
1282 }
1283
1284 static int
1285 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1286     struct iscsi_daemon_handoff *handoff)
1287 {
1288         struct iscsi_session *is;
1289         int error;
1290
1291         sx_slock(&sc->sc_lock);
1292
1293         /*
1294          * Find the session to hand off socket to.
1295          */
1296         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1297                 if (is->is_id == handoff->idh_session_id)
1298                         break;
1299         }
1300         if (is == NULL) {
1301                 sx_sunlock(&sc->sc_lock);
1302                 return (ESRCH);
1303         }
1304         ISCSI_SESSION_LOCK(is);
1305         if (is->is_conf.isc_discovery || is->is_terminating) {
1306                 ISCSI_SESSION_UNLOCK(is);
1307                 sx_sunlock(&sc->sc_lock);
1308                 return (EINVAL);
1309         }
1310         if (is->is_connected) {
1311                 /*
1312                  * This might have happened because another iscsid(8)
1313                  * instance handed off the connection in the meantime.
1314                  * Just return.
1315                  */
1316                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1317                     "session");
1318                 ISCSI_SESSION_UNLOCK(is);
1319                 sx_sunlock(&sc->sc_lock);
1320                 return (EBUSY);
1321         }
1322
1323         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1324             sizeof(is->is_target_alias));
1325         is->is_tsih = handoff->idh_tsih;
1326         is->is_statsn = handoff->idh_statsn;
1327         is->is_initial_r2t = handoff->idh_initial_r2t;
1328         is->is_immediate_data = handoff->idh_immediate_data;
1329         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1330         is->is_max_burst_length = handoff->idh_max_burst_length;
1331         is->is_first_burst_length = handoff->idh_first_burst_length;
1332
1333         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1334                 is->is_conn->ic_header_crc32c = true;
1335         else
1336                 is->is_conn->ic_header_crc32c = false;
1337         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1338                 is->is_conn->ic_data_crc32c = true;
1339         else
1340                 is->is_conn->ic_data_crc32c = false;
1341
1342         is->is_cmdsn = 0;
1343         is->is_expcmdsn = 0;
1344         is->is_maxcmdsn = 0;
1345         is->is_waiting_for_iscsid = false;
1346         is->is_login_phase = false;
1347         is->is_timeout = 0;
1348         is->is_connected = true;
1349         is->is_reason[0] = '\0';
1350
1351         ISCSI_SESSION_UNLOCK(is);
1352
1353 #ifdef ICL_KERNEL_PROXY
1354         if (handoff->idh_socket != 0) {
1355 #endif
1356                 /*
1357                  * Handoff without using ICL proxy.
1358                  */
1359                 error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1360                 if (error != 0) {
1361                         sx_sunlock(&sc->sc_lock);
1362                         iscsi_session_terminate(is);
1363                         return (error);
1364                 }
1365 #ifdef ICL_KERNEL_PROXY
1366         }
1367 #endif
1368
1369         sx_sunlock(&sc->sc_lock);
1370
1371         if (is->is_sim != NULL) {
1372                 /*
1373                  * When reconnecting, there already is SIM allocated for the session.
1374                  */
1375                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1376                 ISCSI_SESSION_LOCK(is);
1377                 ISCSI_SESSION_DEBUG(is, "releasing");
1378                 xpt_release_simq(is->is_sim, 1);
1379                 is->is_simq_frozen = false;
1380                 ISCSI_SESSION_UNLOCK(is);
1381
1382         } else {
1383                 ISCSI_SESSION_LOCK(is);
1384                 is->is_devq = cam_simq_alloc(maxtags);
1385                 if (is->is_devq == NULL) {
1386                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1387                         iscsi_session_terminate(is);
1388                         return (ENOMEM);
1389                 }
1390
1391                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1392                     is, is->is_id /* unit */, &is->is_lock,
1393                     1, maxtags, is->is_devq);
1394                 if (is->is_sim == NULL) {
1395                         ISCSI_SESSION_UNLOCK(is);
1396                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1397                         cam_simq_free(is->is_devq);
1398                         iscsi_session_terminate(is);
1399                         return (ENOMEM);
1400                 }
1401
1402                 error = xpt_bus_register(is->is_sim, NULL, 0);
1403                 if (error != 0) {
1404                         ISCSI_SESSION_UNLOCK(is);
1405                         ISCSI_SESSION_WARN(is, "failed to register bus");
1406                         iscsi_session_terminate(is);
1407                         return (ENOMEM);
1408                 }
1409
1410                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1411                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1412                     CAM_LUN_WILDCARD);
1413                 if (error != CAM_REQ_CMP) {
1414                         ISCSI_SESSION_UNLOCK(is);
1415                         ISCSI_SESSION_WARN(is, "failed to create path");
1416                         iscsi_session_terminate(is);
1417                         return (ENOMEM);
1418                 }
1419                 ISCSI_SESSION_UNLOCK(is);
1420         }
1421
1422         return (0);
1423 }
1424
1425 static int
1426 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1427     struct iscsi_daemon_fail *fail)
1428 {
1429         struct iscsi_session *is;
1430
1431         sx_slock(&sc->sc_lock);
1432
1433         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1434                 if (is->is_id == fail->idf_session_id)
1435                         break;
1436         }
1437         if (is == NULL) {
1438                 sx_sunlock(&sc->sc_lock);
1439                 return (ESRCH);
1440         }
1441         ISCSI_SESSION_LOCK(is);
1442         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1443             fail->idf_reason);
1444         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1445         //is->is_waiting_for_iscsid = false;
1446         //is->is_login_phase = true;
1447         //iscsi_session_reconnect(is);
1448         ISCSI_SESSION_UNLOCK(is);
1449         sx_sunlock(&sc->sc_lock);
1450
1451         return (0);
1452 }
1453
1454 #ifdef ICL_KERNEL_PROXY
1455 static int
1456 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1457     struct iscsi_daemon_connect *idc)
1458 {
1459         struct iscsi_session *is;
1460         struct sockaddr *from_sa, *to_sa;
1461         int error;
1462
1463         sx_slock(&sc->sc_lock);
1464         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1465                 if (is->is_id == idc->idc_session_id)
1466                         break;
1467         }
1468         if (is == NULL) {
1469                 sx_sunlock(&sc->sc_lock);
1470                 return (ESRCH);
1471         }
1472         sx_sunlock(&sc->sc_lock);
1473
1474         if (idc->idc_from_addrlen > 0) {
1475                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1476                 if (error != 0) {
1477                         ISCSI_SESSION_WARN(is,
1478                             "getsockaddr failed with error %d", error);
1479                         return (error);
1480                 }
1481         } else {
1482                 from_sa = NULL;
1483         }
1484         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1485         if (error != 0) {
1486                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1487                     error);
1488                 free(from_sa, M_SONAME);
1489                 return (error);
1490         }
1491
1492         ISCSI_SESSION_LOCK(is);
1493         is->is_waiting_for_iscsid = false;
1494         is->is_login_phase = true;
1495         is->is_timeout = 0;
1496         ISCSI_SESSION_UNLOCK(is);
1497
1498         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1499             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1500         free(from_sa, M_SONAME);
1501         free(to_sa, M_SONAME);
1502
1503         /*
1504          * Digests are always disabled during login phase.
1505          */
1506         is->is_conn->ic_header_crc32c = false;
1507         is->is_conn->ic_data_crc32c = false;
1508
1509         return (error);
1510 }
1511
1512 static int
1513 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1514     struct iscsi_daemon_send *ids)
1515 {
1516         struct iscsi_session *is;
1517         struct icl_pdu *ip;
1518         size_t datalen;
1519         void *data;
1520         int error;
1521
1522         sx_slock(&sc->sc_lock);
1523         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1524                 if (is->is_id == ids->ids_session_id)
1525                         break;
1526         }
1527         if (is == NULL) {
1528                 sx_sunlock(&sc->sc_lock);
1529                 return (ESRCH);
1530         }
1531         sx_sunlock(&sc->sc_lock);
1532
1533         if (is->is_login_phase == false)
1534                 return (EBUSY);
1535
1536         if (is->is_terminating || is->is_reconnecting)
1537                 return (EIO);
1538
1539         datalen = ids->ids_data_segment_len;
1540         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1541                 return (EINVAL);
1542         if (datalen > 0) {
1543                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1544                 error = copyin(ids->ids_data_segment, data, datalen);
1545                 if (error != 0) {
1546                         free(data, M_ISCSI);
1547                         return (error);
1548                 }
1549         }
1550
1551         ip = icl_pdu_new_bhs(is->is_conn, M_WAITOK);
1552         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1553         if (datalen > 0) {
1554                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1555                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1556                 free(data, M_ISCSI);
1557         }
1558         icl_pdu_queue(ip);
1559
1560         return (0);
1561 }
1562
1563 static int
1564 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1565     struct iscsi_daemon_receive *idr)
1566 {
1567         struct iscsi_session *is;
1568         struct icl_pdu *ip;
1569         void *data;
1570
1571         sx_slock(&sc->sc_lock);
1572         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1573                 if (is->is_id == idr->idr_session_id)
1574                         break;
1575         }
1576         if (is == NULL) {
1577                 sx_sunlock(&sc->sc_lock);
1578                 return (ESRCH);
1579         }
1580         sx_sunlock(&sc->sc_lock);
1581
1582         if (is->is_login_phase == false)
1583                 return (EBUSY);
1584
1585         ISCSI_SESSION_LOCK(is);
1586         while (is->is_login_pdu == NULL &&
1587             is->is_terminating == false &&
1588             is->is_reconnecting == false)
1589                 cv_wait(&is->is_login_cv, &is->is_lock);
1590         if (is->is_terminating || is->is_reconnecting) {
1591                 ISCSI_SESSION_UNLOCK(is);
1592                 return (EIO);
1593         }
1594         ip = is->is_login_pdu;
1595         is->is_login_pdu = NULL;
1596         ISCSI_SESSION_UNLOCK(is);
1597
1598         if (ip->ip_data_len > idr->idr_data_segment_len) {
1599                 icl_pdu_free(ip);
1600                 return (EMSGSIZE);
1601         }
1602
1603         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1604         if (ip->ip_data_len > 0) {
1605                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1606                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1607                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1608                 free(data, M_ISCSI);
1609         }
1610
1611         icl_pdu_free(ip);
1612
1613         return (0);
1614 }
1615 #endif /* ICL_KERNEL_PROXY */
1616
1617 static void
1618 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1619 {
1620         /*
1621          * Just make sure all the fields are null-terminated.
1622          *
1623          * XXX: This is not particularly secure.  We should
1624          *      create our own conf and then copy in relevant
1625          *      fields.
1626          */
1627         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1628         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1629         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1630         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1631         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1632         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1633         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1634         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1635         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1636 }
1637
1638 static int
1639 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1640 {
1641         struct iscsi_session *is;
1642         const struct iscsi_session *is2;
1643         int error;
1644
1645         iscsi_sanitize_session_conf(&isa->isa_conf);
1646
1647         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1648         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1649
1650         if (is->is_conf.isc_initiator[0] == '\0' ||
1651             is->is_conf.isc_target_addr[0] == '\0') {
1652                 free(is, M_ISCSI);
1653                 return (EINVAL);
1654         }
1655
1656         if ((is->is_conf.isc_discovery != 0 && is->is_conf.isc_target[0] != 0) ||
1657             (is->is_conf.isc_discovery == 0 && is->is_conf.isc_target[0] == 0)) {
1658                 free(is, M_ISCSI);
1659                 return (EINVAL);
1660         }
1661
1662         sx_xlock(&sc->sc_lock);
1663
1664         /*
1665          * Prevent duplicates.
1666          */
1667         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1668                 if (!!is->is_conf.isc_discovery !=
1669                     !!is2->is_conf.isc_discovery)
1670                         continue;
1671
1672                 if (strcmp(is->is_conf.isc_target_addr,
1673                     is2->is_conf.isc_target_addr) != 0)
1674                         continue;
1675
1676                 if (is->is_conf.isc_discovery == 0 &&
1677                     strcmp(is->is_conf.isc_target,
1678                     is2->is_conf.isc_target) != 0)
1679                         continue;
1680
1681                 sx_xunlock(&sc->sc_lock);
1682                 free(is, M_ISCSI);
1683                 return (EBUSY);
1684         }
1685
1686         is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1687         is->is_conn->ic_receive = iscsi_receive_callback;
1688         is->is_conn->ic_error = iscsi_error_callback;
1689         is->is_conn->ic_prv0 = is;
1690         TAILQ_INIT(&is->is_outstanding);
1691         STAILQ_INIT(&is->is_postponed);
1692         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1693         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1694 #ifdef ICL_KERNEL_PROXY
1695         cv_init(&is->is_login_cv, "iscsi_login");
1696 #endif
1697
1698         is->is_softc = sc;
1699         sc->sc_last_session_id++;
1700         is->is_id = sc->sc_last_session_id;
1701         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1702         arc4rand(&is->is_isid[1], 5, 0);
1703         is->is_tsih = 0;
1704         callout_init(&is->is_callout, 1);
1705         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1706         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1707
1708         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1709         if (error != 0) {
1710                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1711                 return (error);
1712         }
1713
1714         /*
1715          * Trigger immediate reconnection.
1716          */
1717         ISCSI_SESSION_LOCK(is);
1718         is->is_waiting_for_iscsid = true;
1719         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1720         ISCSI_SESSION_UNLOCK(is);
1721         cv_signal(&sc->sc_cv);
1722
1723         sx_xunlock(&sc->sc_lock);
1724
1725         return (0);
1726 }
1727
1728 static bool
1729 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1730     unsigned int id2, const struct iscsi_session_conf *c2)
1731 {
1732         if (id2 == 0 && c2->isc_target[0] == '\0' &&
1733             c2->isc_target_addr[0] == '\0')
1734                 return (true);
1735         if (id2 != 0 && id2 == id1)
1736                 return (true);
1737         if (c2->isc_target[0] != '\0' &&
1738             strcmp(c1->isc_target, c2->isc_target) == 0)
1739                 return (true);
1740         if (c2->isc_target_addr[0] != '\0' &&
1741             strcmp(c1->isc_target_addr, c2->isc_target_addr) == 0)
1742                 return (true);
1743         return (false);
1744 }
1745
1746 static int
1747 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1748     struct iscsi_session_remove *isr)
1749 {
1750         struct iscsi_session *is, *tmp;
1751         bool found = false;
1752
1753         iscsi_sanitize_session_conf(&isr->isr_conf);
1754
1755         sx_xlock(&sc->sc_lock);
1756         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1757                 ISCSI_SESSION_LOCK(is);
1758                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1759                     isr->isr_session_id, &isr->isr_conf)) {
1760                         found = true;
1761                         iscsi_session_logout(is);
1762                         iscsi_session_terminate(is);
1763                 }
1764                 ISCSI_SESSION_UNLOCK(is);
1765         }
1766         sx_xunlock(&sc->sc_lock);
1767
1768         if (!found)
1769                 return (ESRCH);
1770
1771         return (0);
1772 }
1773
1774 static int
1775 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1776 {
1777         int error;
1778         unsigned int i = 0;
1779         struct iscsi_session *is;
1780         struct iscsi_session_state iss;
1781
1782         sx_slock(&sc->sc_lock);
1783         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1784                 if (i >= isl->isl_nentries) {
1785                         sx_sunlock(&sc->sc_lock);
1786                         return (EMSGSIZE);
1787                 }
1788                 memset(&iss, 0, sizeof(iss));
1789                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1790                 iss.iss_id = is->is_id;
1791                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1792                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1793
1794                 if (is->is_conn->ic_header_crc32c)
1795                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1796                 else
1797                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1798
1799                 if (is->is_conn->ic_data_crc32c)
1800                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1801                 else
1802                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1803
1804                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1805                 iss.iss_immediate_data = is->is_immediate_data;
1806                 iss.iss_connected = is->is_connected;
1807
1808                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1809                 if (error != 0) {
1810                         sx_sunlock(&sc->sc_lock);
1811                         return (error);
1812                 }
1813                 i++;
1814         }
1815         sx_sunlock(&sc->sc_lock);
1816
1817         isl->isl_nentries = i;
1818
1819         return (0);
1820 }
1821
1822 static int
1823 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1824     struct thread *td)
1825 {
1826         struct iscsi_softc *sc;
1827
1828         sc = dev->si_drv1;
1829
1830         switch (cmd) {
1831         case ISCSIDWAIT:
1832                 return (iscsi_ioctl_daemon_wait(sc,
1833                     (struct iscsi_daemon_request *)arg));
1834         case ISCSIDHANDOFF:
1835                 return (iscsi_ioctl_daemon_handoff(sc,
1836                     (struct iscsi_daemon_handoff *)arg));
1837         case ISCSIDFAIL:
1838                 return (iscsi_ioctl_daemon_fail(sc,
1839                     (struct iscsi_daemon_fail *)arg));
1840 #ifdef ICL_KERNEL_PROXY
1841         case ISCSIDCONNECT:
1842                 return (iscsi_ioctl_daemon_connect(sc,
1843                     (struct iscsi_daemon_connect *)arg));
1844         case ISCSIDSEND:
1845                 return (iscsi_ioctl_daemon_send(sc,
1846                     (struct iscsi_daemon_send *)arg));
1847         case ISCSIDRECEIVE:
1848                 return (iscsi_ioctl_daemon_receive(sc,
1849                     (struct iscsi_daemon_receive *)arg));
1850 #endif /* ICL_KERNEL_PROXY */
1851         case ISCSISADD:
1852                 return (iscsi_ioctl_session_add(sc,
1853                     (struct iscsi_session_add *)arg));
1854         case ISCSISREMOVE:
1855                 return (iscsi_ioctl_session_remove(sc,
1856                     (struct iscsi_session_remove *)arg));
1857         case ISCSISLIST:
1858                 return (iscsi_ioctl_session_list(sc,
1859                     (struct iscsi_session_list *)arg));
1860         default:
1861                 return (EINVAL);
1862         }
1863 }
1864
1865 static uint64_t
1866 iscsi_encode_lun(uint32_t lun)
1867 {
1868         uint8_t encoded[8];
1869         uint64_t result;
1870
1871         memset(encoded, 0, sizeof(encoded));
1872
1873         if (lun < 256) {
1874                 /*
1875                  * Peripheral device addressing.
1876                  */
1877                 encoded[1] = lun;
1878         } else if (lun < 16384) {
1879                 /*
1880                  * Flat space addressing.
1881                  */
1882                 encoded[0] = 0x40;
1883                 encoded[0] |= (lun >> 8) & 0x3f;
1884                 encoded[1] = lun & 0xff;
1885         } else {
1886                 /*
1887                  * Extended flat space addressing.
1888                  */
1889                 encoded[0] = 0xd2;
1890                 encoded[1] = lun >> 16;
1891                 encoded[2] = lun >> 8;
1892                 encoded[3] = lun;
1893         }
1894
1895         memcpy(&result, encoded, sizeof(result));
1896         return (result);
1897 }
1898
1899 static struct iscsi_outstanding *
1900 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1901 {
1902         struct iscsi_outstanding *io;
1903
1904         ISCSI_SESSION_LOCK_ASSERT(is);
1905
1906         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1907                 if (io->io_initiator_task_tag == initiator_task_tag)
1908                         return (io);
1909         }
1910         return (NULL);
1911 }
1912
1913 static struct iscsi_outstanding *
1914 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
1915 {
1916         struct iscsi_outstanding *io;
1917
1918         ISCSI_SESSION_LOCK_ASSERT(is);
1919
1920         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1921                 if (io->io_ccb == ccb)
1922                         return (io);
1923         }
1924         return (NULL);
1925 }
1926
1927 static struct iscsi_outstanding *
1928 iscsi_outstanding_add(struct iscsi_session *is,
1929     uint32_t initiator_task_tag, union ccb *ccb)
1930 {
1931         struct iscsi_outstanding *io;
1932
1933         ISCSI_SESSION_LOCK_ASSERT(is);
1934
1935         KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
1936             ("initiator_task_tag 0x%x already added", initiator_task_tag));
1937
1938         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
1939         if (io == NULL) {
1940                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
1941                 return (NULL);
1942         }
1943         io->io_initiator_task_tag = initiator_task_tag;
1944         io->io_ccb = ccb;
1945         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
1946         return (io);
1947 }
1948
1949 static void
1950 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
1951 {
1952
1953         ISCSI_SESSION_LOCK_ASSERT(is);
1954
1955         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
1956         uma_zfree(iscsi_outstanding_zone, io);
1957 }
1958
1959 static void
1960 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
1961 {
1962         struct icl_pdu *request;
1963         struct iscsi_bhs_task_management_request *bhstmr;
1964         struct ccb_abort *cab = &ccb->cab;
1965         struct iscsi_outstanding *io, *aio;
1966
1967         ISCSI_SESSION_LOCK_ASSERT(is);
1968
1969 #if 0
1970         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
1971 #else
1972         if (is->is_login_phase) {
1973                 ccb->ccb_h.status = CAM_REQ_ABORTED;
1974                 xpt_done(ccb);
1975                 return;
1976         }
1977 #endif
1978
1979         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
1980         if (aio == NULL) {
1981                 ccb->ccb_h.status = CAM_REQ_CMP;
1982                 xpt_done(ccb);
1983                 return;
1984         }
1985
1986         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
1987         if (request == NULL) {
1988                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
1989                 xpt_done(ccb);
1990                 return;
1991         }
1992
1993         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
1994         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
1995         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
1996
1997         bhstmr->bhstmr_lun = iscsi_encode_lun(ccb->ccb_h.target_lun);
1998         bhstmr->bhstmr_initiator_task_tag = is->is_initiator_task_tag;
1999         is->is_initiator_task_tag++;
2000         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2001
2002         io = iscsi_outstanding_add(is, bhstmr->bhstmr_initiator_task_tag, NULL);
2003         if (io == NULL) {
2004                 icl_pdu_free(request);
2005                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2006                 xpt_done(ccb);
2007                 return;
2008         }
2009         io->io_datasn = aio->io_initiator_task_tag;
2010         iscsi_pdu_queue_locked(request);
2011 }
2012
2013 static void
2014 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2015 {
2016         struct icl_pdu *request;
2017         struct iscsi_bhs_scsi_command *bhssc;
2018         struct ccb_scsiio *csio;
2019         struct iscsi_outstanding *io;
2020         size_t len;
2021         int error;
2022
2023         ISCSI_SESSION_LOCK_ASSERT(is);
2024
2025 #if 0
2026         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2027 #else
2028         if (is->is_login_phase) {
2029                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2030                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2031                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2032                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2033                 }
2034                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2035                 xpt_done(ccb);
2036                 return;
2037         }
2038 #endif
2039
2040         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
2041         if (request == NULL) {
2042                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2043                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2044                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2045                 }
2046                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2047                 xpt_done(ccb);
2048                 return;
2049         }
2050
2051         csio = &ccb->csio;
2052         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2053         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2054         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2055         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2056         case CAM_DIR_IN:
2057                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2058                 break;
2059         case CAM_DIR_OUT:
2060                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2061                 break;
2062         }
2063
2064         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2065                 switch (csio->tag_action) {
2066                 case MSG_HEAD_OF_Q_TAG:
2067                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2068                         break;
2069                 case MSG_ORDERED_Q_TAG:
2070                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2071                         break;
2072                 case MSG_ACA_TASK:
2073                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2074                         break;
2075                 case MSG_SIMPLE_Q_TAG:
2076                 default:
2077                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2078                         break;
2079                 }
2080         } else
2081                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2082
2083         bhssc->bhssc_lun = iscsi_encode_lun(csio->ccb_h.target_lun);
2084         bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
2085         is->is_initiator_task_tag++;
2086         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2087         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2088             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2089
2090         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2091                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2092         else
2093                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2094
2095         io = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
2096         if (io == NULL) {
2097                 icl_pdu_free(request);
2098                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2099                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2100                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2101                 }
2102                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2103                 xpt_done(ccb);
2104                 return;
2105         }
2106
2107         if (is->is_immediate_data &&
2108             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2109                 len = csio->dxfer_len;
2110                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2111                 if (len > is->is_first_burst_length) {
2112                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
2113                         len = is->is_first_burst_length;
2114                 }
2115
2116                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2117                 if (error != 0) {
2118                         icl_pdu_free(request);
2119                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2120                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2121                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2122                         }
2123                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2124                         xpt_done(ccb);
2125                         return;
2126                 }
2127         }
2128         iscsi_pdu_queue_locked(request);
2129 }
2130
2131 static void
2132 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2133 {
2134         struct iscsi_session *is;
2135
2136         is = cam_sim_softc(sim);
2137
2138         ISCSI_SESSION_LOCK_ASSERT(is);
2139
2140         if (is->is_terminating ||
2141             (is->is_connected == false && fail_on_disconnection)) {
2142                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2143                 xpt_done(ccb);
2144                 return;
2145         }
2146
2147         switch (ccb->ccb_h.func_code) {
2148         case XPT_PATH_INQ:
2149         {
2150                 struct ccb_pathinq *cpi = &ccb->cpi;
2151
2152                 cpi->version_num = 1;
2153                 cpi->hba_inquiry = PI_TAG_ABLE;
2154                 cpi->target_sprt = 0;
2155                 //cpi->hba_misc = PIM_NOBUSRESET;
2156                 cpi->hba_misc = 0;
2157                 cpi->hba_eng_cnt = 0;
2158                 cpi->max_target = 0;
2159                 cpi->max_lun = 255;
2160                 //cpi->initiator_id = 0; /* XXX */
2161                 cpi->initiator_id = 64; /* XXX */
2162                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2163                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2164                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2165                 cpi->unit_number = cam_sim_unit(sim);
2166                 cpi->bus_id = cam_sim_bus(sim);
2167                 cpi->base_transfer_speed = 150000; /* XXX */
2168                 cpi->transport = XPORT_ISCSI;
2169                 cpi->transport_version = 0;
2170                 cpi->protocol = PROTO_SCSI;
2171                 cpi->protocol_version = SCSI_REV_SPC3;
2172                 cpi->maxio = MAXPHYS;
2173                 cpi->ccb_h.status = CAM_REQ_CMP;
2174                 break;
2175         }
2176         case XPT_GET_TRAN_SETTINGS:
2177         {
2178                 struct ccb_trans_settings       *cts;
2179                 struct ccb_trans_settings_scsi  *scsi;
2180
2181                 cts = &ccb->cts;
2182                 scsi = &cts->proto_specific.scsi;
2183
2184                 cts->protocol = PROTO_SCSI;
2185                 cts->protocol_version = SCSI_REV_SPC3;
2186                 cts->transport = XPORT_ISCSI;
2187                 cts->transport_version = 0;
2188                 scsi->valid = CTS_SCSI_VALID_TQ;
2189                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2190                 cts->ccb_h.status = CAM_REQ_CMP;
2191                 break;
2192         }
2193         case XPT_CALC_GEOMETRY:
2194                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2195                 ccb->ccb_h.status = CAM_REQ_CMP;
2196                 break;
2197 #if 0
2198         /*
2199          * XXX: What's the point?
2200          */
2201         case XPT_RESET_BUS:
2202         case XPT_TERM_IO:
2203                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2204                 ccb->ccb_h.status = CAM_REQ_CMP;
2205                 break;
2206 #endif
2207         case XPT_ABORT:
2208                 iscsi_action_abort(is, ccb);
2209                 return;
2210         case XPT_SCSI_IO:
2211                 iscsi_action_scsiio(is, ccb);
2212                 return;
2213         default:
2214 #if 0
2215                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2216 #endif
2217                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2218                 break;
2219         }
2220         xpt_done(ccb);
2221 }
2222
2223 static void
2224 iscsi_poll(struct cam_sim *sim)
2225 {
2226
2227         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2228 }
2229
2230 static void
2231 iscsi_shutdown(struct iscsi_softc *sc)
2232 {
2233         struct iscsi_session *is;
2234
2235         ISCSI_DEBUG("removing all sessions due to shutdown");
2236
2237         sx_slock(&sc->sc_lock);
2238         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2239                 iscsi_session_terminate(is);
2240         sx_sunlock(&sc->sc_lock);
2241 }
2242
2243 static int
2244 iscsi_load(void)
2245 {
2246         int error;
2247
2248         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2249         sx_init(&sc->sc_lock, "iscsi");
2250         TAILQ_INIT(&sc->sc_sessions);
2251         cv_init(&sc->sc_cv, "iscsi_cv");
2252
2253         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2254             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2255             UMA_ALIGN_PTR, 0);
2256
2257         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2258             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2259         if (error != 0) {
2260                 ISCSI_WARN("failed to create device node, error %d", error);
2261                 return (error);
2262         }
2263         sc->sc_cdev->si_drv1 = sc;
2264
2265         /*
2266          * Note that this needs to get run before dashutdown().  Otherwise,
2267          * when rebooting with iSCSI session with outstanding requests,
2268          * but disconnected, dashutdown() will hang on cam_periph_runccb().
2269          */
2270         sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2271             iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2272
2273         return (0);
2274 }
2275
2276 static int
2277 iscsi_unload(void)
2278 {
2279         struct iscsi_session *is, *tmp;
2280
2281         if (sc->sc_cdev != NULL) {
2282                 ISCSI_DEBUG("removing device node");
2283                 destroy_dev(sc->sc_cdev);
2284                 ISCSI_DEBUG("device node removed");
2285         }
2286
2287         if (sc->sc_shutdown_eh != NULL)
2288                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2289
2290         sx_slock(&sc->sc_lock);
2291         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2292                 iscsi_session_terminate(is);
2293         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2294                 ISCSI_DEBUG("waiting for sessions to terminate");
2295                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2296         }
2297         ISCSI_DEBUG("all sessions terminated");
2298         sx_sunlock(&sc->sc_lock);
2299
2300         uma_zdestroy(iscsi_outstanding_zone);
2301         sx_destroy(&sc->sc_lock);
2302         cv_destroy(&sc->sc_cv);
2303         free(sc, M_ISCSI);
2304         return (0);
2305 }
2306
2307 static int
2308 iscsi_quiesce(void)
2309 {
2310         sx_slock(&sc->sc_lock);
2311         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2312                 sx_sunlock(&sc->sc_lock);
2313                 return (EBUSY);
2314         }
2315         sx_sunlock(&sc->sc_lock);
2316         return (0);
2317 }
2318
2319 static int
2320 iscsi_modevent(module_t mod, int what, void *arg)
2321 {
2322         int error;
2323
2324         switch (what) {
2325         case MOD_LOAD:
2326                 error = iscsi_load();
2327                 break;
2328         case MOD_UNLOAD:
2329                 error = iscsi_unload();
2330                 break;
2331         case MOD_QUIESCE:
2332                 error = iscsi_quiesce();
2333                 break;
2334         default:
2335                 error = EINVAL;
2336                 break;
2337         }
2338         return (error);
2339 }
2340
2341 moduledata_t iscsi_data = {
2342         "iscsi",
2343         iscsi_modevent,
2344         0
2345 };
2346
2347 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2348 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2349 MODULE_DEPEND(iscsi, icl, 1, 1, 1);