sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/condvar.h>
  36 #include <sys/conf.h>
  37 #include <sys/eventhandler.h>
  38 #include <sys/file.h>
  39 #include <sys/kernel.h>
  40 #include <sys/kthread.h>
  41 #include <sys/lock.h>
  42 #include <sys/malloc.h>
  43 #include <sys/mutex.h>
  44 #include <sys/module.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/systm.h>
  47 #include <sys/sx.h>
  48 #include <vm/uma.h>
  49
  50 #include <cam/cam.h>
  51 #include <cam/cam_ccb.h>
  52 #include <cam/cam_xpt.h>
  53 #include <cam/cam_debug.h>
  54 #include <cam/cam_sim.h>
  55 #include <cam/cam_xpt_sim.h>
  56 #include <cam/cam_xpt_periph.h>
  57 #include <cam/cam_periph.h>
  58 #include <cam/scsi/scsi_all.h>
  59 #include <cam/scsi/scsi_message.h>
  60
  61 #include <dev/iscsi/icl.h>
  62 #include <dev/iscsi/iscsi_ioctl.h>
  63 #include <dev/iscsi/iscsi_proto.h>
  64 #include <dev/iscsi/iscsi.h>
  65
  66 #ifdef ICL_KERNEL_PROXY
  67 #include <sys/socketvar.h>
  68 #endif
  69
  70 #ifdef ICL_KERNEL_PROXY
  71 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  72 #endif
  73
  74 /*
  75  * XXX: This is global so the iscsi_unload() can access it.
  76  *      Think about how to do this properly.
  77  */
  78 static struct iscsi_softc       *sc;
  79
  80 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  81 static int debug = 1;
  82 TUNABLE_INT("kern.iscsi.debug", &debug);
  83 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  84     &debug, 0, "Enable debug messages");
  85 static int ping_timeout = 5;
  86 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
  87 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  88     0, "Timeout for ping (NOP-Out) requests, in seconds");
  89 static int iscsid_timeout = 60;
  90 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
  91 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  92     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  93 static int login_timeout = 60;
  94 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
  95 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
  96     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  97 static int maxtags = 255;
  98 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
  99 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
 100     0, "Max number of IO requests queued");
 101 static int fail_on_disconnection = 0;
 102 TUNABLE_INT("kern.iscsi.fail_on_disconnection", &fail_on_disconnection);
 103 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
 104     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
 105 static int fail_on_shutdown = 1;
 106 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_shutdown, CTLFLAG_RWTUN,
 107     &fail_on_shutdown, 0, "Fail disconnected sessions on shutdown");
 108
 109 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 110 static uma_zone_t iscsi_outstanding_zone;
 111
 112 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 113 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 114
 115 #define ISCSI_DEBUG(X, ...)                                             \
 116         do {                                                            \
 117                 if (debug > 1)                                          \
 118                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 119         } while (0)
 120
 121 #define ISCSI_WARN(X, ...)                                              \
 122         do {                                                            \
 123                 if (debug > 0) {                                        \
 124                         printf("WARNING: %s: " X "\n",                  \
 125                             __func__, ## __VA_ARGS__);                  \
 126                 }                                                       \
 127         } while (0)
 128
 129 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 130         do {                                                            \
 131                 if (debug > 1) {                                        \
 132                         printf("%s: %s (%s): " X "\n",                  \
 133                             __func__, S->is_conf.isc_target_addr,       \
 134                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 135                 }                                                       \
 136         } while (0)
 137
 138 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 139         do {                                                            \
 140                 if (debug > 0) {                                        \
 141                         printf("WARNING: %s (%s): " X "\n",             \
 142                             S->is_conf.isc_target_addr,                 \
 143                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 144                 }                                                       \
 145         } while (0)
 146
 147 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 148 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 149 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 150 #define ISCSI_SESSION_LOCK_ASSERT_NOT(X) mtx_assert(&X->is_lock, MA_NOTOWNED)
 151
 152 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 153                     int mode, struct thread *td);
 154
 155 static struct cdevsw iscsi_cdevsw = {
 156      .d_version = D_VERSION,
 157      .d_ioctl   = iscsi_ioctl,
 158      .d_name    = "iscsi",
 159 };
 160
 161 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 162 static void     iscsi_pdu_queue(struct icl_pdu *request);
 163 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 164 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 165 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 166 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
 167 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 168 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 169 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 170 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 171 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 172 static void     iscsi_session_reconnect(struct iscsi_session *is);
 173 static void     iscsi_session_terminate(struct iscsi_session *is);
 174 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 175 static void     iscsi_poll(struct cam_sim *sim);
 176 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 177                     uint32_t initiator_task_tag);
 178 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
 179                     uint32_t initiator_task_tag, union ccb *ccb);
 180 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 181                     struct iscsi_outstanding *io);
 182
 183 static bool
 184 iscsi_pdu_prepare(struct icl_pdu *request)
 185 {
 186         struct iscsi_session *is;
 187         struct iscsi_bhs_scsi_command *bhssc;
 188
 189         is = PDU_SESSION(request);
 190
 191         ISCSI_SESSION_LOCK_ASSERT(is);
 192
 193         /*
 194          * We're only using fields common for all the request
 195          * (initiator -> target) PDUs.
 196          */
 197         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 198
 199         /*
 200          * Data-Out PDU does not contain CmdSN.
 201          */
 202         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 203                 if (ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn) &&
 204                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 205                         /*
 206                          * Current MaxCmdSN prevents us from sending any more
 207                          * SCSI Command PDUs to the target; postpone the PDU.
 208                          * It will get resent by either iscsi_pdu_queue(),
 209                          * or by maintenance thread.
 210                          */
 211 #if 0
 212                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %u, "
 213                             "ExpCmdSN %u, MaxCmdSN %u, opcode 0x%x",
 214                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn,
 215                             bhssc->bhssc_opcode);
 216 #endif
 217                         return (true);
 218                 }
 219                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 220                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 221                         is->is_cmdsn++;
 222         }
 223         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 224
 225         return (false);
 226 }
 227
 228 static void
 229 iscsi_session_send_postponed(struct iscsi_session *is)
 230 {
 231         struct icl_pdu *request;
 232         bool postpone;
 233
 234         ISCSI_SESSION_LOCK_ASSERT(is);
 235
 236         while (!STAILQ_EMPTY(&is->is_postponed)) {
 237                 request = STAILQ_FIRST(&is->is_postponed);
 238                 postpone = iscsi_pdu_prepare(request);
 239                 if (postpone)
 240                         break;
 241                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 242                 icl_pdu_queue(request);
 243         }
 244 }
 245
 246 static void
 247 iscsi_pdu_queue_locked(struct icl_pdu *request)
 248 {
 249         struct iscsi_session *is;
 250         bool postpone;
 251
 252         is = PDU_SESSION(request);
 253         ISCSI_SESSION_LOCK_ASSERT(is);
 254         iscsi_session_send_postponed(is);
 255         postpone = iscsi_pdu_prepare(request);
 256         if (postpone) {
 257                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 258                 return;
 259         }
 260         icl_pdu_queue(request);
 261 }
 262
 263 static void
 264 iscsi_pdu_queue(struct icl_pdu *request)
 265 {
 266         struct iscsi_session *is;
 267
 268         is = PDU_SESSION(request);
 269         ISCSI_SESSION_LOCK(is);
 270         iscsi_pdu_queue_locked(request);
 271         ISCSI_SESSION_UNLOCK(is);
 272 }
 273
 274 static void
 275 iscsi_session_logout(struct iscsi_session *is)
 276 {
 277         struct icl_pdu *request;
 278         struct iscsi_bhs_logout_request *bhslr;
 279
 280         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 281         if (request == NULL)
 282                 return;
 283
 284         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 285         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 286         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 287         iscsi_pdu_queue_locked(request);
 288 }
 289
 290 static void
 291 iscsi_session_terminate_task(struct iscsi_session *is,
 292     struct iscsi_outstanding *io, bool requeue)
 293 {
 294
 295         if (io->io_ccb != NULL) {
 296                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
 297                 if (requeue)
 298                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 299                 else
 300                         io->io_ccb->ccb_h.status |= CAM_REQ_ABORTED;
 301                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 302                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 303                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 304                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 305                 }
 306                 xpt_done(io->io_ccb);
 307         }
 308         iscsi_outstanding_remove(is, io);
 309 }
 310
 311 static void
 312 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 313 {
 314         struct iscsi_outstanding *io, *tmp;
 315
 316         ISCSI_SESSION_LOCK_ASSERT(is);
 317
 318         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 319                 iscsi_session_terminate_task(is, io, requeue);
 320         }
 321 }
 322
 323 static void
 324 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 325 {
 326         struct icl_pdu *pdu;
 327
 328         ISCSI_SESSION_LOCK_ASSERT(is);
 329
 330         /*
 331          * Don't queue any new PDUs.
 332          */
 333         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 334                 ISCSI_SESSION_DEBUG(is, "freezing");
 335                 xpt_freeze_simq(is->is_sim, 1);
 336                 is->is_simq_frozen = true;
 337         }
 338
 339         /*
 340          * Remove postponed PDUs.
 341          */
 342         while (!STAILQ_EMPTY(&is->is_postponed)) {
 343                 pdu = STAILQ_FIRST(&is->is_postponed);
 344                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 345                 icl_pdu_free(pdu);
 346         }
 347
 348         if (destroy_sim == false) {
 349                 /*
 350                  * Terminate SCSI tasks, asking CAM to requeue them.
 351                  */
 352                 iscsi_session_terminate_tasks(is, true);
 353                 return;
 354         }
 355
 356         iscsi_session_terminate_tasks(is, false);
 357
 358         if (is->is_sim == NULL)
 359                 return;
 360
 361         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 362         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 363
 364         if (is->is_simq_frozen) {
 365                 xpt_release_simq(is->is_sim, 1);
 366                 is->is_simq_frozen = false;
 367         }
 368
 369         xpt_free_path(is->is_path);
 370         is->is_path = NULL;
 371         xpt_bus_deregister(cam_sim_path(is->is_sim));
 372         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 373         is->is_sim = NULL;
 374         is->is_devq = NULL;
 375 }
 376
 377 static void
 378 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 379 {
 380
 381         icl_conn_close(is->is_conn);
 382
 383         ISCSI_SESSION_LOCK(is);
 384
 385         is->is_connected = false;
 386         is->is_reconnecting = false;
 387         is->is_login_phase = false;
 388
 389 #ifdef ICL_KERNEL_PROXY
 390         if (is->is_login_pdu != NULL) {
 391                 icl_pdu_free(is->is_login_pdu);
 392                 is->is_login_pdu = NULL;
 393         }
 394         cv_signal(&is->is_login_cv);
 395 #endif
 396
 397         if (fail_on_disconnection) {
 398                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
 399                 iscsi_session_cleanup(is, true);
 400         } else {
 401                 iscsi_session_cleanup(is, false);
 402         }
 403
 404         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 405             ("destroying session with active tasks"));
 406         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 407             ("destroying session with postponed PDUs"));
 408
 409         /*
 410          * Request immediate reconnection from iscsid(8).
 411          */
 412         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 413         is->is_waiting_for_iscsid = true;
 414         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 415         is->is_timeout = 0;
 416         ISCSI_SESSION_UNLOCK(is);
 417         cv_signal(&is->is_softc->sc_cv);
 418 }
 419
 420 static void
 421 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 422 {
 423         struct iscsi_softc *sc;
 424
 425         sc = is->is_softc;
 426         sx_xlock(&sc->sc_lock);
 427
 428         icl_conn_close(is->is_conn);
 429         callout_drain(&is->is_callout);
 430
 431         ISCSI_SESSION_LOCK(is);
 432
 433         KASSERT(is->is_terminating, ("is_terminating == false"));
 434
 435 #ifdef ICL_KERNEL_PROXY
 436         if (is->is_login_pdu != NULL) {
 437                 icl_pdu_free(is->is_login_pdu);
 438                 is->is_login_pdu = NULL;
 439         }
 440         cv_signal(&is->is_login_cv);
 441 #endif
 442
 443         iscsi_session_cleanup(is, true);
 444
 445         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 446             ("destroying session with active tasks"));
 447         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 448             ("destroying session with postponed PDUs"));
 449
 450         ISCSI_SESSION_UNLOCK(is);
 451
 452         icl_conn_free(is->is_conn);
 453         mtx_destroy(&is->is_lock);
 454         cv_destroy(&is->is_maintenance_cv);
 455 #ifdef ICL_KERNEL_PROXY
 456         cv_destroy(&is->is_login_cv);
 457 #endif
 458         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 459         sx_xunlock(&sc->sc_lock);
 460
 461         ISCSI_SESSION_DEBUG(is, "terminated");
 462         free(is, M_ISCSI);
 463
 464         /*
 465          * The iscsi_unload() routine might be waiting.
 466          */
 467         cv_signal(&sc->sc_cv);
 468 }
 469
 470 static void
 471 iscsi_maintenance_thread(void *arg)
 472 {
 473         struct iscsi_session *is;
 474
 475         is = arg;
 476
 477         for (;;) {
 478                 ISCSI_SESSION_LOCK(is);
 479                 if (is->is_reconnecting == false &&
 480                     is->is_terminating == false &&
 481                     STAILQ_EMPTY(&is->is_postponed))
 482                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 483
 484                 /* Terminate supersedes reconnect. */
 485                 if (is->is_terminating) {
 486                         ISCSI_SESSION_UNLOCK(is);
 487                         iscsi_maintenance_thread_terminate(is);
 488                         kthread_exit();
 489                         return;
 490                 }
 491
 492                 if (is->is_reconnecting) {
 493                         ISCSI_SESSION_UNLOCK(is);
 494                         iscsi_maintenance_thread_reconnect(is);
 495                         continue;
 496                 }
 497
 498                 iscsi_session_send_postponed(is);
 499                 ISCSI_SESSION_UNLOCK(is);
 500         }
 501 }
 502
 503 static void
 504 iscsi_session_reconnect(struct iscsi_session *is)
 505 {
 506
 507         /*
 508          * XXX: We can't use locking here, because
 509          *      it's being called from various contexts.
 510          *      Hope it doesn't break anything.
 511          */
 512         if (is->is_reconnecting)
 513                 return;
 514
 515         is->is_reconnecting = true;
 516         cv_signal(&is->is_maintenance_cv);
 517 }
 518
 519 static void
 520 iscsi_session_terminate(struct iscsi_session *is)
 521 {
 522
 523         if (is->is_terminating)
 524                 return;
 525
 526         is->is_terminating = true;
 527
 528 #if 0
 529         iscsi_session_logout(is);
 530 #endif
 531         cv_signal(&is->is_maintenance_cv);
 532 }
 533
 534 static void
 535 iscsi_callout(void *context)
 536 {
 537         struct icl_pdu *request;
 538         struct iscsi_bhs_nop_out *bhsno;
 539         struct iscsi_session *is;
 540         bool reconnect_needed = false;
 541
 542         is = context;
 543
 544         ISCSI_SESSION_LOCK(is);
 545         if (is->is_terminating) {
 546                 ISCSI_SESSION_UNLOCK(is);
 547                 return;
 548         }
 549
 550         callout_schedule(&is->is_callout, 1 * hz);
 551
 552         is->is_timeout++;
 553
 554         if (is->is_waiting_for_iscsid) {
 555                 if (iscsid_timeout > 0 && is->is_timeout > iscsid_timeout) {
 556                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 557                             "for %d seconds; reconnecting",
 558                             is->is_timeout);
 559                         reconnect_needed = true;
 560                 }
 561                 goto out;
 562         }
 563
 564         if (is->is_login_phase) {
 565                 if (login_timeout > 0 && is->is_timeout > login_timeout) {
 566                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 567                             "reconnecting", is->is_timeout);
 568                         reconnect_needed = true;
 569                 }
 570                 goto out;
 571         }
 572
 573         if (ping_timeout <= 0) {
 574                 /*
 575                  * Pings are disabled.  Don't send NOP-Out in this case.
 576                  * Reset the timeout, to avoid triggering reconnection,
 577                  * should the user decide to reenable them.
 578                  */
 579                 is->is_timeout = 0;
 580                 goto out;
 581         }
 582
 583         if (is->is_timeout >= ping_timeout) {
 584                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 585                     "reconnecting", ping_timeout);
 586                 reconnect_needed = true;
 587                 goto out;
 588         }
 589
 590         ISCSI_SESSION_UNLOCK(is);
 591
 592         /*
 593          * If the ping was reset less than one second ago - which means
 594          * that we've received some PDU during the last second - assume
 595          * the traffic flows correctly and don't bother sending a NOP-Out.
 596          *
 597          * (It's 2 - one for one second, and one for incrementing is_timeout
 598          * earlier in this routine.)
 599          */
 600         if (is->is_timeout < 2)
 601                 return;
 602
 603         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 604         if (request == NULL) {
 605                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 606                 return;
 607         }
 608         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 609         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 610             ISCSI_BHS_OPCODE_IMMEDIATE;
 611         bhsno->bhsno_flags = 0x80;
 612         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 613         iscsi_pdu_queue(request);
 614         return;
 615
 616 out:
 617         if (is->is_terminating) {
 618                 ISCSI_SESSION_UNLOCK(is);
 619                 return;
 620         }
 621
 622         ISCSI_SESSION_UNLOCK(is);
 623
 624         if (reconnect_needed)
 625                 iscsi_session_reconnect(is);
 626 }
 627
 628 static void
 629 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 630 {
 631         const struct iscsi_bhs_data_in *bhsdi;
 632         struct iscsi_session *is;
 633         uint32_t expcmdsn, maxcmdsn, statsn;
 634
 635         is = PDU_SESSION(response);
 636
 637         ISCSI_SESSION_LOCK_ASSERT(is);
 638
 639         /*
 640          * We're only using fields common for all the response
 641          * (target -> initiator) PDUs.
 642          */
 643         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 644         /*
 645          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 646          * and Residual Count only have meaningful content if the S bit
 647          * is set to 1", so we also need to check the bit specific for
 648          * Data-In PDU.
 649          */
 650         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 651             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 652                 statsn = ntohl(bhsdi->bhsdi_statsn);
 653                 if (statsn != is->is_statsn && statsn != (is->is_statsn + 1)) {
 654                         /* XXX: This is normal situation for MCS */
 655                         ISCSI_SESSION_WARN(is, "PDU 0x%x StatSN %u != "
 656                             "session ExpStatSN %u (or + 1); reconnecting",
 657                             bhsdi->bhsdi_opcode, statsn, is->is_statsn);
 658                         iscsi_session_reconnect(is);
 659                 }
 660                 if (ISCSI_SNGT(statsn, is->is_statsn))
 661                         is->is_statsn = statsn;
 662         }
 663
 664         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 665         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 666
 667         if (ISCSI_SNLT(maxcmdsn + 1, expcmdsn)) {
 668                 ISCSI_SESSION_DEBUG(is,
 669                     "PDU MaxCmdSN %u + 1 < PDU ExpCmdSN %u; ignoring",
 670                     maxcmdsn, expcmdsn);
 671         } else {
 672                 if (ISCSI_SNGT(maxcmdsn, is->is_maxcmdsn)) {
 673                         is->is_maxcmdsn = maxcmdsn;
 674
 675                         /*
 676                          * Command window increased; kick the maintanance thread
 677                          * to send out postponed commands.
 678                          */
 679                         if (!STAILQ_EMPTY(&is->is_postponed))
 680                                 cv_signal(&is->is_maintenance_cv);
 681                 } else if (ISCSI_SNLT(maxcmdsn, is->is_maxcmdsn)) {
 682                         /* XXX: This is normal situation for MCS */
 683                         ISCSI_SESSION_DEBUG(is,
 684                             "PDU MaxCmdSN %u < session MaxCmdSN %u; ignoring",
 685                             maxcmdsn, is->is_maxcmdsn);
 686                 }
 687
 688                 if (ISCSI_SNGT(expcmdsn, is->is_expcmdsn)) {
 689                         is->is_expcmdsn = expcmdsn;
 690                 } else if (ISCSI_SNLT(expcmdsn, is->is_expcmdsn)) {
 691                         /* XXX: This is normal situation for MCS */
 692                         ISCSI_SESSION_DEBUG(is,
 693                             "PDU ExpCmdSN %u < session ExpCmdSN %u; ignoring",
 694                             expcmdsn, is->is_expcmdsn);
 695                 }
 696         }
 697
 698         /*
 699          * Every incoming PDU - not just NOP-In - resets the ping timer.
 700          * The purpose of the timeout is to reset the connection when it stalls;
 701          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 702          * in some queue.
 703          */
 704         is->is_timeout = 0;
 705 }
 706
 707 static void
 708 iscsi_receive_callback(struct icl_pdu *response)
 709 {
 710         struct iscsi_session *is;
 711
 712         is = PDU_SESSION(response);
 713
 714         ISCSI_SESSION_LOCK(is);
 715
 716 #ifdef ICL_KERNEL_PROXY
 717         if (is->is_login_phase) {
 718                 if (is->is_login_pdu == NULL)
 719                         is->is_login_pdu = response;
 720                 else
 721                         icl_pdu_free(response);
 722                 ISCSI_SESSION_UNLOCK(is);
 723                 cv_signal(&is->is_login_cv);
 724                 return;
 725         }
 726 #endif
 727
 728         iscsi_pdu_update_statsn(response);
 729
 730         /*
 731          * The handling routine is responsible for freeing the PDU
 732          * when it's no longer needed.
 733          */
 734         switch (response->ip_bhs->bhs_opcode) {
 735         case ISCSI_BHS_OPCODE_NOP_IN:
 736                 iscsi_pdu_handle_nop_in(response);
 737                 ISCSI_SESSION_UNLOCK(is);
 738                 break;
 739         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 740                 iscsi_pdu_handle_scsi_response(response);
 741                 /* Session lock dropped inside. */
 742                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 743                 break;
 744         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
 745                 iscsi_pdu_handle_task_response(response);
 746                 ISCSI_SESSION_UNLOCK(is);
 747                 break;
 748         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 749                 iscsi_pdu_handle_data_in(response);
 750                 /* Session lock dropped inside. */
 751                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 752                 break;
 753         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 754                 iscsi_pdu_handle_logout_response(response);
 755                 ISCSI_SESSION_UNLOCK(is);
 756                 break;
 757         case ISCSI_BHS_OPCODE_R2T:
 758                 iscsi_pdu_handle_r2t(response);
 759                 ISCSI_SESSION_UNLOCK(is);
 760                 break;
 761         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 762                 iscsi_pdu_handle_async_message(response);
 763                 ISCSI_SESSION_UNLOCK(is);
 764                 break;
 765         case ISCSI_BHS_OPCODE_REJECT:
 766                 iscsi_pdu_handle_reject(response);
 767                 ISCSI_SESSION_UNLOCK(is);
 768                 break;
 769         default:
 770                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 771                     "opcode 0x%x; reconnecting",
 772                     response->ip_bhs->bhs_opcode);
 773                 iscsi_session_reconnect(is);
 774                 ISCSI_SESSION_UNLOCK(is);
 775                 icl_pdu_free(response);
 776         }
 777 }
 778
 779 static void
 780 iscsi_error_callback(struct icl_conn *ic)
 781 {
 782         struct iscsi_session *is;
 783
 784         is = CONN_SESSION(ic);
 785
 786         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 787         iscsi_session_reconnect(is);
 788 }
 789
 790 static void
 791 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 792 {
 793         struct iscsi_session *is;
 794         struct iscsi_bhs_nop_out *bhsno;
 795         struct iscsi_bhs_nop_in *bhsni;
 796         struct icl_pdu *request;
 797         void *data = NULL;
 798         size_t datasize;
 799         int error;
 800
 801         is = PDU_SESSION(response);
 802         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 803
 804         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 805                 /*
 806                  * Nothing to do; iscsi_pdu_update_statsn() already
 807                  * zeroed the timeout.
 808                  */
 809                 icl_pdu_free(response);
 810                 return;
 811         }
 812
 813         datasize = icl_pdu_data_segment_length(response);
 814         if (datasize > 0) {
 815                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 816                 if (data == NULL) {
 817                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 818                             "reconnecting");
 819                         icl_pdu_free(response);
 820                         iscsi_session_reconnect(is);
 821                         return;
 822                 }
 823                 icl_pdu_get_data(response, 0, data, datasize);
 824         }
 825
 826         request = icl_pdu_new(response->ip_conn, M_NOWAIT);
 827         if (request == NULL) {
 828                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 829                     "reconnecting");
 830                 free(data, M_ISCSI);
 831                 icl_pdu_free(response);
 832                 iscsi_session_reconnect(is);
 833                 return;
 834         }
 835         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 836         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 837             ISCSI_BHS_OPCODE_IMMEDIATE;
 838         bhsno->bhsno_flags = 0x80;
 839         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 840         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 841         if (datasize > 0) {
 842                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 843                 if (error != 0) {
 844                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 845                             "reconnecting");
 846                         free(data, M_ISCSI);
 847                         icl_pdu_free(request);
 848                         icl_pdu_free(response);
 849                         iscsi_session_reconnect(is);
 850                         return;
 851                 }
 852                 free(data, M_ISCSI);
 853         }
 854
 855         icl_pdu_free(response);
 856         iscsi_pdu_queue_locked(request);
 857 }
 858
 859 static void
 860 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 861 {
 862         struct iscsi_bhs_scsi_response *bhssr;
 863         struct iscsi_outstanding *io;
 864         struct iscsi_session *is;
 865         union ccb *ccb;
 866         struct ccb_scsiio *csio;
 867         size_t data_segment_len, received;
 868         uint16_t sense_len;
 869
 870         is = PDU_SESSION(response);
 871
 872         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 873         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 874         if (io == NULL || io->io_ccb == NULL) {
 875                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 876                 icl_pdu_free(response);
 877                 iscsi_session_reconnect(is);
 878                 ISCSI_SESSION_UNLOCK(is);
 879                 return;
 880         }
 881
 882         ccb = io->io_ccb;
 883         received = io->io_received;
 884         iscsi_outstanding_remove(is, io);
 885         ISCSI_SESSION_UNLOCK(is);
 886
 887         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 888                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 889                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 890                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 891                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 892                 }
 893                 ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 894         } else if (bhssr->bhssr_status == 0) {
 895                 ccb->ccb_h.status = CAM_REQ_CMP;
 896         } else {
 897                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 898                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 899                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 900                 }
 901                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 902                 ccb->csio.scsi_status = bhssr->bhssr_status;
 903         }
 904
 905         csio = &ccb->csio;
 906         data_segment_len = icl_pdu_data_segment_length(response);
 907         if (data_segment_len > 0) {
 908                 if (data_segment_len < sizeof(sense_len)) {
 909                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 910                             data_segment_len);
 911                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 912                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 913                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 914                         }
 915                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 916                         goto out;
 917                 }
 918                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 919                 sense_len = ntohs(sense_len);
 920 #if 0
 921                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 922                     sense_len, data_segment_len);
 923 #endif
 924                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 925                         ISCSI_SESSION_WARN(is, "truncated data segment "
 926                             "(%zd bytes, should be %zd)",
 927                             data_segment_len, sizeof(sense_len) + sense_len);
 928                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 929                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 930                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 931                         }
 932                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 933                         goto out;
 934                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 935                         ISCSI_SESSION_WARN(is, "oversize data segment "
 936                             "(%zd bytes, should be %zd)",
 937                             data_segment_len, sizeof(sense_len) + sense_len);
 938                 if (sense_len > csio->sense_len) {
 939                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 940                             sense_len, csio->sense_len);
 941                         sense_len = csio->sense_len;
 942                 }
 943                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 944                 csio->sense_resid = csio->sense_len - sense_len;
 945                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 946         }
 947
 948 out:
 949         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 950                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 951
 952         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 953                 KASSERT(received <= csio->dxfer_len,
 954                     ("received > csio->dxfer_len"));
 955                 if (received < csio->dxfer_len) {
 956                         if (csio->resid != csio->dxfer_len - received) {
 957                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 958                                     "target indicates %d, we calculated %zd",
 959                                     csio->resid, csio->dxfer_len - received);
 960                         }
 961                         csio->resid = csio->dxfer_len - received;
 962                 }
 963         }
 964
 965         xpt_done(ccb);
 966         icl_pdu_free(response);
 967 }
 968
 969 static void
 970 iscsi_pdu_handle_task_response(struct icl_pdu *response)
 971 {
 972         struct iscsi_bhs_task_management_response *bhstmr;
 973         struct iscsi_outstanding *io, *aio;
 974         struct iscsi_session *is;
 975
 976         is = PDU_SESSION(response);
 977
 978         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
 979         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
 980         if (io == NULL || io->io_ccb != NULL) {
 981                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
 982                     bhstmr->bhstmr_initiator_task_tag);
 983                 icl_pdu_free(response);
 984                 iscsi_session_reconnect(is);
 985                 return;
 986         }
 987
 988         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
 989                 ISCSI_SESSION_WARN(is, "task response 0x%x",
 990                     bhstmr->bhstmr_response);
 991         } else {
 992                 aio = iscsi_outstanding_find(is, io->io_datasn);
 993                 if (aio != NULL && aio->io_ccb != NULL)
 994                         iscsi_session_terminate_task(is, aio, false);
 995         }
 996
 997         iscsi_outstanding_remove(is, io);
 998         icl_pdu_free(response);
 999 }
1000
1001 static void
1002 iscsi_pdu_handle_data_in(struct icl_pdu *response)
1003 {
1004         struct iscsi_bhs_data_in *bhsdi;
1005         struct iscsi_outstanding *io;
1006         struct iscsi_session *is;
1007         union ccb *ccb;
1008         struct ccb_scsiio *csio;
1009         size_t data_segment_len, received, oreceived;
1010
1011         is = PDU_SESSION(response);
1012         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
1013         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
1014         if (io == NULL || io->io_ccb == NULL) {
1015                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
1016                 icl_pdu_free(response);
1017                 iscsi_session_reconnect(is);
1018                 ISCSI_SESSION_UNLOCK(is);
1019                 return;
1020         }
1021
1022         data_segment_len = icl_pdu_data_segment_length(response);
1023         if (data_segment_len == 0) {
1024                 /*
1025                  * "The sending of 0 length data segments should be avoided,
1026                  * but initiators and targets MUST be able to properly receive
1027                  * 0 length data segments."
1028                  */
1029                 ISCSI_SESSION_UNLOCK(is);
1030                 icl_pdu_free(response);
1031                 return;
1032         }
1033
1034         /*
1035          * We need to track this for security reasons - without it, malicious target
1036          * could respond to SCSI READ without sending Data-In PDUs, which would result
1037          * in read operation on the initiator side returning random kernel data.
1038          */
1039         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1040                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1041                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1042                 icl_pdu_free(response);
1043                 iscsi_session_reconnect(is);
1044                 ISCSI_SESSION_UNLOCK(is);
1045                 return;
1046         }
1047
1048         ccb = io->io_ccb;
1049         csio = &ccb->csio;
1050
1051         if (io->io_received + data_segment_len > csio->dxfer_len) {
1052                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1053                     "at offset %zd, buffer is %d)",
1054                     data_segment_len, io->io_received, csio->dxfer_len);
1055                 icl_pdu_free(response);
1056                 iscsi_session_reconnect(is);
1057                 ISCSI_SESSION_UNLOCK(is);
1058                 return;
1059         }
1060
1061         oreceived = io->io_received;
1062         io->io_received += data_segment_len;
1063         received = io->io_received;
1064         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0)
1065                 iscsi_outstanding_remove(is, io);
1066         ISCSI_SESSION_UNLOCK(is);
1067
1068         icl_pdu_get_data(response, 0, csio->data_ptr + oreceived, data_segment_len);
1069
1070         /*
1071          * XXX: Check DataSN.
1072          * XXX: Check F.
1073          */
1074         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1075                 /*
1076                  * Nothing more to do.
1077                  */
1078                 icl_pdu_free(response);
1079                 return;
1080         }
1081
1082         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1083         if (bhsdi->bhsdi_status == 0) {
1084                 ccb->ccb_h.status = CAM_REQ_CMP;
1085         } else {
1086                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1087                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1088                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1089                 }
1090                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1091                 csio->scsi_status = bhsdi->bhsdi_status;
1092         }
1093
1094         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1095                 KASSERT(received <= csio->dxfer_len,
1096                     ("received > csio->dxfer_len"));
1097                 if (received < csio->dxfer_len) {
1098                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1099                         if (csio->resid != csio->dxfer_len - received) {
1100                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1101                                     "target indicates %d, we calculated %zd",
1102                                     csio->resid, csio->dxfer_len - received);
1103                         }
1104                         csio->resid = csio->dxfer_len - received;
1105                 }
1106         }
1107
1108         xpt_done(ccb);
1109         icl_pdu_free(response);
1110 }
1111
1112 static void
1113 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1114 {
1115
1116         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1117         icl_pdu_free(response);
1118 }
1119
1120 static void
1121 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1122 {
1123         struct icl_pdu *request;
1124         struct iscsi_session *is;
1125         struct iscsi_bhs_r2t *bhsr2t;
1126         struct iscsi_bhs_data_out *bhsdo;
1127         struct iscsi_outstanding *io;
1128         struct ccb_scsiio *csio;
1129         size_t off, len, total_len;
1130         int error;
1131
1132         is = PDU_SESSION(response);
1133
1134         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1135         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1136         if (io == NULL || io->io_ccb == NULL) {
1137                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1138                     bhsr2t->bhsr2t_initiator_task_tag);
1139                 icl_pdu_free(response);
1140                 iscsi_session_reconnect(is);
1141                 return;
1142         }
1143
1144         csio = &io->io_ccb->csio;
1145
1146         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1147                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1148                 icl_pdu_free(response);
1149                 iscsi_session_reconnect(is);
1150                 return;
1151         }
1152
1153         /*
1154          * XXX: Verify R2TSN.
1155          */
1156
1157         io->io_datasn = 0;
1158
1159         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1160         if (off > csio->dxfer_len) {
1161                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1162                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1163                 icl_pdu_free(response);
1164                 iscsi_session_reconnect(is);
1165                 return;
1166         }
1167
1168         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1169         if (total_len == 0 || total_len > csio->dxfer_len) {
1170                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1171                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1172                 icl_pdu_free(response);
1173                 iscsi_session_reconnect(is);
1174                 return;
1175         }
1176
1177         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1178
1179         for (;;) {
1180                 len = total_len;
1181
1182                 if (len > is->is_max_data_segment_length)
1183                         len = is->is_max_data_segment_length;
1184
1185                 if (off + len > csio->dxfer_len) {
1186                         ISCSI_SESSION_WARN(is, "target requested invalid "
1187                             "length/offset %zd, buffer is %d; reconnecting",
1188                             off + len, csio->dxfer_len);
1189                         icl_pdu_free(response);
1190                         iscsi_session_reconnect(is);
1191                         return;
1192                 }
1193
1194                 request = icl_pdu_new(response->ip_conn, M_NOWAIT);
1195                 if (request == NULL) {
1196                         icl_pdu_free(response);
1197                         iscsi_session_reconnect(is);
1198                         return;
1199                 }
1200
1201                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1202                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1203                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1204                 bhsdo->bhsdo_initiator_task_tag =
1205                     bhsr2t->bhsr2t_initiator_task_tag;
1206                 bhsdo->bhsdo_target_transfer_tag =
1207                     bhsr2t->bhsr2t_target_transfer_tag;
1208                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1209                 bhsdo->bhsdo_buffer_offset = htonl(off);
1210                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1211                     M_NOWAIT);
1212                 if (error != 0) {
1213                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1214                             "reconnecting");
1215                         icl_pdu_free(request);
1216                         icl_pdu_free(response);
1217                         iscsi_session_reconnect(is);
1218                         return;
1219                 }
1220
1221                 off += len;
1222                 total_len -= len;
1223
1224                 if (total_len == 0) {
1225                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1226                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1227                 } else {
1228                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1229                 }
1230
1231                 iscsi_pdu_queue_locked(request);
1232
1233                 if (total_len == 0)
1234                         break;
1235         }
1236
1237         icl_pdu_free(response);
1238 }
1239
1240 static void
1241 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1242 {
1243         struct iscsi_bhs_asynchronous_message *bhsam;
1244         struct iscsi_session *is;
1245
1246         is = PDU_SESSION(response);
1247         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1248         switch (bhsam->bhsam_async_event) {
1249         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1250                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1251                 iscsi_session_logout(is);
1252                 iscsi_session_terminate(is);
1253                 break;
1254         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1255                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1256                 break;
1257         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1258                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1259                 break;
1260         default:
1261                 /*
1262                  * XXX: Technically, we're obligated to also handle
1263                  *      parameter renegotiation.
1264                  */
1265                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1266                 break;
1267         }
1268
1269         icl_pdu_free(response);
1270 }
1271
1272 static void
1273 iscsi_pdu_handle_reject(struct icl_pdu *response)
1274 {
1275         struct iscsi_bhs_reject *bhsr;
1276         struct iscsi_session *is;
1277
1278         is = PDU_SESSION(response);
1279         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1280         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1281             bhsr->bhsr_reason);
1282
1283         icl_pdu_free(response);
1284 }
1285
1286 static int
1287 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1288     struct iscsi_daemon_request *request)
1289 {
1290         struct iscsi_session *is;
1291         int error;
1292
1293         sx_slock(&sc->sc_lock);
1294         for (;;) {
1295                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1296                         ISCSI_SESSION_LOCK(is);
1297                         if (is->is_waiting_for_iscsid)
1298                                 break;
1299                         ISCSI_SESSION_UNLOCK(is);
1300                 }
1301
1302                 if (is == NULL) {
1303                         /*
1304                          * No session requires attention from iscsid(8); wait.
1305                          */
1306                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1307                         if (error != 0) {
1308                                 sx_sunlock(&sc->sc_lock);
1309                                 return (error);
1310                         }
1311                         continue;
1312                 }
1313
1314                 is->is_waiting_for_iscsid = false;
1315                 is->is_login_phase = true;
1316                 is->is_reason[0] = '\0';
1317                 ISCSI_SESSION_UNLOCK(is);
1318
1319                 request->idr_session_id = is->is_id;
1320                 memcpy(&request->idr_isid, &is->is_isid,
1321                     sizeof(request->idr_isid));
1322                 request->idr_tsih = 0;  /* New or reinstated session. */
1323                 memcpy(&request->idr_conf, &is->is_conf,
1324                     sizeof(request->idr_conf));
1325
1326                 sx_sunlock(&sc->sc_lock);
1327                 return (0);
1328         }
1329 }
1330
1331 static int
1332 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1333     struct iscsi_daemon_handoff *handoff)
1334 {
1335         struct iscsi_session *is;
1336         int error;
1337
1338         sx_slock(&sc->sc_lock);
1339
1340         /*
1341          * Find the session to hand off socket to.
1342          */
1343         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1344                 if (is->is_id == handoff->idh_session_id)
1345                         break;
1346         }
1347         if (is == NULL) {
1348                 sx_sunlock(&sc->sc_lock);
1349                 return (ESRCH);
1350         }
1351         ISCSI_SESSION_LOCK(is);
1352         if (is->is_conf.isc_discovery || is->is_terminating) {
1353                 ISCSI_SESSION_UNLOCK(is);
1354                 sx_sunlock(&sc->sc_lock);
1355                 return (EINVAL);
1356         }
1357         if (is->is_connected) {
1358                 /*
1359                  * This might have happened because another iscsid(8)
1360                  * instance handed off the connection in the meantime.
1361                  * Just return.
1362                  */
1363                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1364                     "session");
1365                 ISCSI_SESSION_UNLOCK(is);
1366                 sx_sunlock(&sc->sc_lock);
1367                 return (EBUSY);
1368         }
1369
1370         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1371             sizeof(is->is_target_alias));
1372         is->is_tsih = handoff->idh_tsih;
1373         is->is_statsn = handoff->idh_statsn;
1374         is->is_initial_r2t = handoff->idh_initial_r2t;
1375         is->is_immediate_data = handoff->idh_immediate_data;
1376         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1377         is->is_max_burst_length = handoff->idh_max_burst_length;
1378         is->is_first_burst_length = handoff->idh_first_burst_length;
1379
1380         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1381                 is->is_conn->ic_header_crc32c = true;
1382         else
1383                 is->is_conn->ic_header_crc32c = false;
1384         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1385                 is->is_conn->ic_data_crc32c = true;
1386         else
1387                 is->is_conn->ic_data_crc32c = false;
1388
1389         is->is_cmdsn = 0;
1390         is->is_expcmdsn = 0;
1391         is->is_maxcmdsn = 0;
1392         is->is_waiting_for_iscsid = false;
1393         is->is_login_phase = false;
1394         is->is_timeout = 0;
1395         is->is_connected = true;
1396         is->is_reason[0] = '\0';
1397
1398         ISCSI_SESSION_UNLOCK(is);
1399
1400 #ifdef ICL_KERNEL_PROXY
1401         if (handoff->idh_socket != 0) {
1402 #endif
1403                 /*
1404                  * Handoff without using ICL proxy.
1405                  */
1406                 error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1407                 if (error != 0) {
1408                         sx_sunlock(&sc->sc_lock);
1409                         iscsi_session_terminate(is);
1410                         return (error);
1411                 }
1412 #ifdef ICL_KERNEL_PROXY
1413         }
1414 #endif
1415
1416         sx_sunlock(&sc->sc_lock);
1417
1418         if (is->is_sim != NULL) {
1419                 /*
1420                  * When reconnecting, there already is SIM allocated for the session.
1421                  */
1422                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1423                 ISCSI_SESSION_LOCK(is);
1424                 ISCSI_SESSION_DEBUG(is, "releasing");
1425                 xpt_release_simq(is->is_sim, 1);
1426                 is->is_simq_frozen = false;
1427                 ISCSI_SESSION_UNLOCK(is);
1428
1429         } else {
1430                 ISCSI_SESSION_LOCK(is);
1431                 is->is_devq = cam_simq_alloc(maxtags);
1432                 if (is->is_devq == NULL) {
1433                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1434                         iscsi_session_terminate(is);
1435                         return (ENOMEM);
1436                 }
1437
1438                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1439                     is, is->is_id /* unit */, &is->is_lock,
1440                     1, maxtags, is->is_devq);
1441                 if (is->is_sim == NULL) {
1442                         ISCSI_SESSION_UNLOCK(is);
1443                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1444                         cam_simq_free(is->is_devq);
1445                         iscsi_session_terminate(is);
1446                         return (ENOMEM);
1447                 }
1448
1449                 error = xpt_bus_register(is->is_sim, NULL, 0);
1450                 if (error != 0) {
1451                         ISCSI_SESSION_UNLOCK(is);
1452                         ISCSI_SESSION_WARN(is, "failed to register bus");
1453                         iscsi_session_terminate(is);
1454                         return (ENOMEM);
1455                 }
1456
1457                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1458                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1459                     CAM_LUN_WILDCARD);
1460                 if (error != CAM_REQ_CMP) {
1461                         ISCSI_SESSION_UNLOCK(is);
1462                         ISCSI_SESSION_WARN(is, "failed to create path");
1463                         iscsi_session_terminate(is);
1464                         return (ENOMEM);
1465                 }
1466                 ISCSI_SESSION_UNLOCK(is);
1467         }
1468
1469         return (0);
1470 }
1471
1472 static int
1473 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1474     struct iscsi_daemon_fail *fail)
1475 {
1476         struct iscsi_session *is;
1477
1478         sx_slock(&sc->sc_lock);
1479
1480         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1481                 if (is->is_id == fail->idf_session_id)
1482                         break;
1483         }
1484         if (is == NULL) {
1485                 sx_sunlock(&sc->sc_lock);
1486                 return (ESRCH);
1487         }
1488         ISCSI_SESSION_LOCK(is);
1489         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1490             fail->idf_reason);
1491         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1492         //is->is_waiting_for_iscsid = false;
1493         //is->is_login_phase = true;
1494         //iscsi_session_reconnect(is);
1495         ISCSI_SESSION_UNLOCK(is);
1496         sx_sunlock(&sc->sc_lock);
1497
1498         return (0);
1499 }
1500
1501 #ifdef ICL_KERNEL_PROXY
1502 static int
1503 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1504     struct iscsi_daemon_connect *idc)
1505 {
1506         struct iscsi_session *is;
1507         struct sockaddr *from_sa, *to_sa;
1508         int error;
1509
1510         sx_slock(&sc->sc_lock);
1511         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1512                 if (is->is_id == idc->idc_session_id)
1513                         break;
1514         }
1515         if (is == NULL) {
1516                 sx_sunlock(&sc->sc_lock);
1517                 return (ESRCH);
1518         }
1519         sx_sunlock(&sc->sc_lock);
1520
1521         if (idc->idc_from_addrlen > 0) {
1522                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1523                 if (error != 0) {
1524                         ISCSI_SESSION_WARN(is,
1525                             "getsockaddr failed with error %d", error);
1526                         return (error);
1527                 }
1528         } else {
1529                 from_sa = NULL;
1530         }
1531         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1532         if (error != 0) {
1533                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1534                     error);
1535                 free(from_sa, M_SONAME);
1536                 return (error);
1537         }
1538
1539         ISCSI_SESSION_LOCK(is);
1540         is->is_waiting_for_iscsid = false;
1541         is->is_login_phase = true;
1542         is->is_timeout = 0;
1543         ISCSI_SESSION_UNLOCK(is);
1544
1545         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1546             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1547         free(from_sa, M_SONAME);
1548         free(to_sa, M_SONAME);
1549
1550         /*
1551          * Digests are always disabled during login phase.
1552          */
1553         is->is_conn->ic_header_crc32c = false;
1554         is->is_conn->ic_data_crc32c = false;
1555
1556         return (error);
1557 }
1558
1559 static int
1560 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1561     struct iscsi_daemon_send *ids)
1562 {
1563         struct iscsi_session *is;
1564         struct icl_pdu *ip;
1565         size_t datalen;
1566         void *data;
1567         int error;
1568
1569         sx_slock(&sc->sc_lock);
1570         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1571                 if (is->is_id == ids->ids_session_id)
1572                         break;
1573         }
1574         if (is == NULL) {
1575                 sx_sunlock(&sc->sc_lock);
1576                 return (ESRCH);
1577         }
1578         sx_sunlock(&sc->sc_lock);
1579
1580         if (is->is_login_phase == false)
1581                 return (EBUSY);
1582
1583         if (is->is_terminating || is->is_reconnecting)
1584                 return (EIO);
1585
1586         datalen = ids->ids_data_segment_len;
1587         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1588                 return (EINVAL);
1589         if (datalen > 0) {
1590                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1591                 error = copyin(ids->ids_data_segment, data, datalen);
1592                 if (error != 0) {
1593                         free(data, M_ISCSI);
1594                         return (error);
1595                 }
1596         }
1597
1598         ip = icl_pdu_new(is->is_conn, M_WAITOK);
1599         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1600         if (datalen > 0) {
1601                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1602                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1603                 free(data, M_ISCSI);
1604         }
1605         icl_pdu_queue(ip);
1606
1607         return (0);
1608 }
1609
1610 static int
1611 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1612     struct iscsi_daemon_receive *idr)
1613 {
1614         struct iscsi_session *is;
1615         struct icl_pdu *ip;
1616         void *data;
1617
1618         sx_slock(&sc->sc_lock);
1619         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1620                 if (is->is_id == idr->idr_session_id)
1621                         break;
1622         }
1623         if (is == NULL) {
1624                 sx_sunlock(&sc->sc_lock);
1625                 return (ESRCH);
1626         }
1627         sx_sunlock(&sc->sc_lock);
1628
1629         if (is->is_login_phase == false)
1630                 return (EBUSY);
1631
1632         ISCSI_SESSION_LOCK(is);
1633         while (is->is_login_pdu == NULL &&
1634             is->is_terminating == false &&
1635             is->is_reconnecting == false)
1636                 cv_wait(&is->is_login_cv, &is->is_lock);
1637         if (is->is_terminating || is->is_reconnecting) {
1638                 ISCSI_SESSION_UNLOCK(is);
1639                 return (EIO);
1640         }
1641         ip = is->is_login_pdu;
1642         is->is_login_pdu = NULL;
1643         ISCSI_SESSION_UNLOCK(is);
1644
1645         if (ip->ip_data_len > idr->idr_data_segment_len) {
1646                 icl_pdu_free(ip);
1647                 return (EMSGSIZE);
1648         }
1649
1650         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1651         if (ip->ip_data_len > 0) {
1652                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1653                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1654                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1655                 free(data, M_ISCSI);
1656         }
1657
1658         icl_pdu_free(ip);
1659
1660         return (0);
1661 }
1662 #endif /* ICL_KERNEL_PROXY */
1663
1664 static void
1665 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1666 {
1667         /*
1668          * Just make sure all the fields are null-terminated.
1669          *
1670          * XXX: This is not particularly secure.  We should
1671          *      create our own conf and then copy in relevant
1672          *      fields.
1673          */
1674         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1675         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1676         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1677         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1678         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1679         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1680         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1681         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1682         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1683 }
1684
1685 static bool
1686 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1687 {
1688
1689         if (isc->isc_initiator[0] == '\0') {
1690                 ISCSI_DEBUG("empty isc_initiator");
1691                 return (false);
1692         }
1693
1694         if (isc->isc_target_addr[0] == '\0') {
1695                 ISCSI_DEBUG("empty isc_target_addr");
1696                 return (false);
1697         }
1698
1699         if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1700                 ISCSI_DEBUG("non-empty isc_target for discovery session");
1701                 return (false);
1702         }
1703
1704         if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1705                 ISCSI_DEBUG("empty isc_target for non-discovery session");
1706                 return (false);
1707         }
1708
1709         return (true);
1710 }
1711
1712 static int
1713 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1714 {
1715         struct iscsi_session *is;
1716         const struct iscsi_session *is2;
1717         int error;
1718
1719         iscsi_sanitize_session_conf(&isa->isa_conf);
1720         if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1721                 return (EINVAL);
1722
1723         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1724         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1725
1726         sx_xlock(&sc->sc_lock);
1727
1728         /*
1729          * Prevent duplicates.
1730          */
1731         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1732                 if (!!is->is_conf.isc_discovery !=
1733                     !!is2->is_conf.isc_discovery)
1734                         continue;
1735
1736                 if (strcmp(is->is_conf.isc_target_addr,
1737                     is2->is_conf.isc_target_addr) != 0)
1738                         continue;
1739
1740                 if (is->is_conf.isc_discovery == 0 &&
1741                     strcmp(is->is_conf.isc_target,
1742                     is2->is_conf.isc_target) != 0)
1743                         continue;
1744
1745                 sx_xunlock(&sc->sc_lock);
1746                 free(is, M_ISCSI);
1747                 return (EBUSY);
1748         }
1749
1750         is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1751         is->is_conn->ic_receive = iscsi_receive_callback;
1752         is->is_conn->ic_error = iscsi_error_callback;
1753         is->is_conn->ic_prv0 = is;
1754         TAILQ_INIT(&is->is_outstanding);
1755         STAILQ_INIT(&is->is_postponed);
1756         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1757         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1758 #ifdef ICL_KERNEL_PROXY
1759         cv_init(&is->is_login_cv, "iscsi_login");
1760 #endif
1761
1762         is->is_softc = sc;
1763         sc->sc_last_session_id++;
1764         is->is_id = sc->sc_last_session_id;
1765         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1766         arc4rand(&is->is_isid[1], 5, 0);
1767         is->is_tsih = 0;
1768         callout_init(&is->is_callout, 1);
1769
1770         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1771         if (error != 0) {
1772                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1773                 sx_xunlock(&sc->sc_lock);
1774                 return (error);
1775         }
1776
1777         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1778         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1779
1780         /*
1781          * Trigger immediate reconnection.
1782          */
1783         ISCSI_SESSION_LOCK(is);
1784         is->is_waiting_for_iscsid = true;
1785         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1786         ISCSI_SESSION_UNLOCK(is);
1787         cv_signal(&sc->sc_cv);
1788
1789         sx_xunlock(&sc->sc_lock);
1790
1791         return (0);
1792 }
1793
1794 static bool
1795 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1796     unsigned int id2, const struct iscsi_session_conf *c2)
1797 {
1798
1799         if (id2 != 0 && id2 != id1)
1800                 return (false);
1801         if (c2->isc_target[0] != '\0' &&
1802             strcmp(c1->isc_target, c2->isc_target) != 0)
1803                 return (false);
1804         if (c2->isc_target_addr[0] != '\0' &&
1805             strcmp(c1->isc_target_addr, c2->isc_target_addr) != 0)
1806                 return (false);
1807         return (true);
1808 }
1809
1810 static int
1811 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1812     struct iscsi_session_remove *isr)
1813 {
1814         struct iscsi_session *is, *tmp;
1815         bool found = false;
1816
1817         iscsi_sanitize_session_conf(&isr->isr_conf);
1818
1819         sx_xlock(&sc->sc_lock);
1820         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1821                 ISCSI_SESSION_LOCK(is);
1822                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1823                     isr->isr_session_id, &isr->isr_conf)) {
1824                         found = true;
1825                         iscsi_session_logout(is);
1826                         iscsi_session_terminate(is);
1827                 }
1828                 ISCSI_SESSION_UNLOCK(is);
1829         }
1830         sx_xunlock(&sc->sc_lock);
1831
1832         if (!found)
1833                 return (ESRCH);
1834
1835         return (0);
1836 }
1837
1838 static int
1839 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1840 {
1841         int error;
1842         unsigned int i = 0;
1843         struct iscsi_session *is;
1844         struct iscsi_session_state iss;
1845
1846         sx_slock(&sc->sc_lock);
1847         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1848                 if (i >= isl->isl_nentries) {
1849                         sx_sunlock(&sc->sc_lock);
1850                         return (EMSGSIZE);
1851                 }
1852                 memset(&iss, 0, sizeof(iss));
1853                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1854                 iss.iss_id = is->is_id;
1855                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1856                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1857
1858                 if (is->is_conn->ic_header_crc32c)
1859                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1860                 else
1861                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1862
1863                 if (is->is_conn->ic_data_crc32c)
1864                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1865                 else
1866                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1867
1868                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1869                 iss.iss_immediate_data = is->is_immediate_data;
1870                 iss.iss_connected = is->is_connected;
1871
1872                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1873                 if (error != 0) {
1874                         sx_sunlock(&sc->sc_lock);
1875                         return (error);
1876                 }
1877                 i++;
1878         }
1879         sx_sunlock(&sc->sc_lock);
1880
1881         isl->isl_nentries = i;
1882
1883         return (0);
1884 }
1885
1886 static int
1887 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
1888     struct iscsi_session_modify *ism)
1889 {
1890         struct iscsi_session *is;
1891
1892         iscsi_sanitize_session_conf(&ism->ism_conf);
1893         if (iscsi_valid_session_conf(&ism->ism_conf) == false)
1894                 return (EINVAL);
1895
1896         sx_xlock(&sc->sc_lock);
1897         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1898                 ISCSI_SESSION_LOCK(is);
1899                 if (is->is_id == ism->ism_session_id)
1900                         break;
1901                 ISCSI_SESSION_UNLOCK(is);
1902         }
1903         if (is == NULL) {
1904                 sx_xunlock(&sc->sc_lock);
1905                 return (ESRCH);
1906         }
1907         sx_xunlock(&sc->sc_lock);
1908
1909         memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
1910         ISCSI_SESSION_UNLOCK(is);
1911
1912         iscsi_session_reconnect(is);
1913
1914         return (0);
1915 }
1916
1917 static int
1918 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1919     struct thread *td)
1920 {
1921         struct iscsi_softc *sc;
1922
1923         sc = dev->si_drv1;
1924
1925         switch (cmd) {
1926         case ISCSIDWAIT:
1927                 return (iscsi_ioctl_daemon_wait(sc,
1928                     (struct iscsi_daemon_request *)arg));
1929         case ISCSIDHANDOFF:
1930                 return (iscsi_ioctl_daemon_handoff(sc,
1931                     (struct iscsi_daemon_handoff *)arg));
1932         case ISCSIDFAIL:
1933                 return (iscsi_ioctl_daemon_fail(sc,
1934                     (struct iscsi_daemon_fail *)arg));
1935 #ifdef ICL_KERNEL_PROXY
1936         case ISCSIDCONNECT:
1937                 return (iscsi_ioctl_daemon_connect(sc,
1938                     (struct iscsi_daemon_connect *)arg));
1939         case ISCSIDSEND:
1940                 return (iscsi_ioctl_daemon_send(sc,
1941                     (struct iscsi_daemon_send *)arg));
1942         case ISCSIDRECEIVE:
1943                 return (iscsi_ioctl_daemon_receive(sc,
1944                     (struct iscsi_daemon_receive *)arg));
1945 #endif /* ICL_KERNEL_PROXY */
1946         case ISCSISADD:
1947                 return (iscsi_ioctl_session_add(sc,
1948                     (struct iscsi_session_add *)arg));
1949         case ISCSISREMOVE:
1950                 return (iscsi_ioctl_session_remove(sc,
1951                     (struct iscsi_session_remove *)arg));
1952         case ISCSISLIST:
1953                 return (iscsi_ioctl_session_list(sc,
1954                     (struct iscsi_session_list *)arg));
1955         case ISCSISMODIFY:
1956                 return (iscsi_ioctl_session_modify(sc,
1957                     (struct iscsi_session_modify *)arg));
1958         default:
1959                 return (EINVAL);
1960         }
1961 }
1962
1963 static uint64_t
1964 iscsi_encode_lun(uint32_t lun)
1965 {
1966         uint8_t encoded[8];
1967         uint64_t result;
1968
1969         memset(encoded, 0, sizeof(encoded));
1970
1971         if (lun < 256) {
1972                 /*
1973                  * Peripheral device addressing.
1974                  */
1975                 encoded[1] = lun;
1976         } else if (lun < 16384) {
1977                 /*
1978                  * Flat space addressing.
1979                  */
1980                 encoded[0] = 0x40;
1981                 encoded[0] |= (lun >> 8) & 0x3f;
1982                 encoded[1] = lun & 0xff;
1983         } else {
1984                 /*
1985                  * Extended flat space addressing.
1986                  */
1987                 encoded[0] = 0xd2;
1988                 encoded[1] = lun >> 16;
1989                 encoded[2] = lun >> 8;
1990                 encoded[3] = lun;
1991         }
1992
1993         memcpy(&result, encoded, sizeof(result));
1994         return (result);
1995 }
1996
1997 static struct iscsi_outstanding *
1998 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1999 {
2000         struct iscsi_outstanding *io;
2001
2002         ISCSI_SESSION_LOCK_ASSERT(is);
2003
2004         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2005                 if (io->io_initiator_task_tag == initiator_task_tag)
2006                         return (io);
2007         }
2008         return (NULL);
2009 }
2010
2011 static struct iscsi_outstanding *
2012 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
2013 {
2014         struct iscsi_outstanding *io;
2015
2016         ISCSI_SESSION_LOCK_ASSERT(is);
2017
2018         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2019                 if (io->io_ccb == ccb)
2020                         return (io);
2021         }
2022         return (NULL);
2023 }
2024
2025 static struct iscsi_outstanding *
2026 iscsi_outstanding_add(struct iscsi_session *is,
2027     uint32_t initiator_task_tag, union ccb *ccb)
2028 {
2029         struct iscsi_outstanding *io;
2030
2031         ISCSI_SESSION_LOCK_ASSERT(is);
2032
2033         KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
2034             ("initiator_task_tag 0x%x already added", initiator_task_tag));
2035
2036         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
2037         if (io == NULL) {
2038                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
2039                 return (NULL);
2040         }
2041         io->io_initiator_task_tag = initiator_task_tag;
2042         io->io_ccb = ccb;
2043         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
2044         return (io);
2045 }
2046
2047 static void
2048 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
2049 {
2050
2051         ISCSI_SESSION_LOCK_ASSERT(is);
2052
2053         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
2054         uma_zfree(iscsi_outstanding_zone, io);
2055 }
2056
2057 static void
2058 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
2059 {
2060         struct icl_pdu *request;
2061         struct iscsi_bhs_task_management_request *bhstmr;
2062         struct ccb_abort *cab = &ccb->cab;
2063         struct iscsi_outstanding *io, *aio;
2064
2065         ISCSI_SESSION_LOCK_ASSERT(is);
2066
2067 #if 0
2068         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2069 #else
2070         if (is->is_login_phase) {
2071                 ccb->ccb_h.status = CAM_REQ_ABORTED;
2072                 xpt_done(ccb);
2073                 return;
2074         }
2075 #endif
2076
2077         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
2078         if (aio == NULL) {
2079                 ccb->ccb_h.status = CAM_REQ_CMP;
2080                 xpt_done(ccb);
2081                 return;
2082         }
2083
2084         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2085         if (request == NULL) {
2086                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2087                 xpt_done(ccb);
2088                 return;
2089         }
2090
2091         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2092         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
2093         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
2094
2095         bhstmr->bhstmr_lun = iscsi_encode_lun(ccb->ccb_h.target_lun);
2096         bhstmr->bhstmr_initiator_task_tag = is->is_initiator_task_tag;
2097         is->is_initiator_task_tag++;
2098         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2099
2100         io = iscsi_outstanding_add(is, bhstmr->bhstmr_initiator_task_tag, NULL);
2101         if (io == NULL) {
2102                 icl_pdu_free(request);
2103                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2104                 xpt_done(ccb);
2105                 return;
2106         }
2107         io->io_datasn = aio->io_initiator_task_tag;
2108         iscsi_pdu_queue_locked(request);
2109 }
2110
2111 static void
2112 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2113 {
2114         struct icl_pdu *request;
2115         struct iscsi_bhs_scsi_command *bhssc;
2116         struct ccb_scsiio *csio;
2117         struct iscsi_outstanding *io;
2118         size_t len;
2119         int error;
2120
2121         ISCSI_SESSION_LOCK_ASSERT(is);
2122
2123 #if 0
2124         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2125 #else
2126         if (is->is_login_phase) {
2127                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2128                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2129                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2130                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2131                 }
2132                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2133                 xpt_done(ccb);
2134                 return;
2135         }
2136 #endif
2137
2138         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2139         if (request == NULL) {
2140                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2141                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2142                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2143                 }
2144                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2145                 xpt_done(ccb);
2146                 return;
2147         }
2148
2149         csio = &ccb->csio;
2150         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2151         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2152         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2153         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2154         case CAM_DIR_IN:
2155                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2156                 break;
2157         case CAM_DIR_OUT:
2158                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2159                 break;
2160         }
2161
2162         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2163                 switch (csio->tag_action) {
2164                 case MSG_HEAD_OF_Q_TAG:
2165                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2166                         break;
2167                 case MSG_ORDERED_Q_TAG:
2168                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2169                         break;
2170                 case MSG_ACA_TASK:
2171                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2172                         break;
2173                 case MSG_SIMPLE_Q_TAG:
2174                 default:
2175                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2176                         break;
2177                 }
2178         } else
2179                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2180
2181         bhssc->bhssc_lun = iscsi_encode_lun(csio->ccb_h.target_lun);
2182         bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
2183         is->is_initiator_task_tag++;
2184         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2185         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2186             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2187
2188         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2189                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2190         else
2191                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2192
2193         io = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
2194         if (io == NULL) {
2195                 icl_pdu_free(request);
2196                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2197                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2198                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2199                 }
2200                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2201                 xpt_done(ccb);
2202                 return;
2203         }
2204
2205         if (is->is_immediate_data &&
2206             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2207                 len = csio->dxfer_len;
2208                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2209                 if (len > is->is_first_burst_length) {
2210                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
2211                         len = is->is_first_burst_length;
2212                 }
2213                 if (len > is->is_max_data_segment_length) {
2214                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_max_data_segment_length);
2215                         len = is->is_max_data_segment_length;
2216                 }
2217
2218                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2219                 if (error != 0) {
2220                         icl_pdu_free(request);
2221                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2222                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2223                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2224                         }
2225                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2226                         xpt_done(ccb);
2227                         return;
2228                 }
2229         }
2230         iscsi_pdu_queue_locked(request);
2231 }
2232
2233 static void
2234 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2235 {
2236         struct iscsi_session *is;
2237
2238         is = cam_sim_softc(sim);
2239
2240         ISCSI_SESSION_LOCK_ASSERT(is);
2241
2242         if (is->is_terminating ||
2243             (is->is_connected == false && fail_on_disconnection)) {
2244                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2245                 xpt_done(ccb);
2246                 return;
2247         }
2248
2249         switch (ccb->ccb_h.func_code) {
2250         case XPT_PATH_INQ:
2251         {
2252                 struct ccb_pathinq *cpi = &ccb->cpi;
2253
2254                 cpi->version_num = 1;
2255                 cpi->hba_inquiry = PI_TAG_ABLE;
2256                 cpi->target_sprt = 0;
2257                 //cpi->hba_misc = PIM_NOBUSRESET;
2258                 cpi->hba_misc = 0;
2259                 cpi->hba_eng_cnt = 0;
2260                 cpi->max_target = 0;
2261                 cpi->max_lun = 255;
2262                 //cpi->initiator_id = 0; /* XXX */
2263                 cpi->initiator_id = 64; /* XXX */
2264                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2265                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2266                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2267                 cpi->unit_number = cam_sim_unit(sim);
2268                 cpi->bus_id = cam_sim_bus(sim);
2269                 cpi->base_transfer_speed = 150000; /* XXX */
2270                 cpi->transport = XPORT_ISCSI;
2271                 cpi->transport_version = 0;
2272                 cpi->protocol = PROTO_SCSI;
2273                 cpi->protocol_version = SCSI_REV_SPC3;
2274                 cpi->maxio = MAXPHYS;
2275                 cpi->ccb_h.status = CAM_REQ_CMP;
2276                 break;
2277         }
2278         case XPT_GET_TRAN_SETTINGS:
2279         {
2280                 struct ccb_trans_settings       *cts;
2281                 struct ccb_trans_settings_scsi  *scsi;
2282
2283                 cts = &ccb->cts;
2284                 scsi = &cts->proto_specific.scsi;
2285
2286                 cts->protocol = PROTO_SCSI;
2287                 cts->protocol_version = SCSI_REV_SPC3;
2288                 cts->transport = XPORT_ISCSI;
2289                 cts->transport_version = 0;
2290                 scsi->valid = CTS_SCSI_VALID_TQ;
2291                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2292                 cts->ccb_h.status = CAM_REQ_CMP;
2293                 break;
2294         }
2295         case XPT_CALC_GEOMETRY:
2296                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2297                 ccb->ccb_h.status = CAM_REQ_CMP;
2298                 break;
2299 #if 0
2300         /*
2301          * XXX: What's the point?
2302          */
2303         case XPT_RESET_BUS:
2304         case XPT_TERM_IO:
2305                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2306                 ccb->ccb_h.status = CAM_REQ_CMP;
2307                 break;
2308 #endif
2309         case XPT_ABORT:
2310                 iscsi_action_abort(is, ccb);
2311                 return;
2312         case XPT_SCSI_IO:
2313                 iscsi_action_scsiio(is, ccb);
2314                 return;
2315         default:
2316 #if 0
2317                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2318 #endif
2319                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2320                 break;
2321         }
2322         xpt_done(ccb);
2323 }
2324
2325 static void
2326 iscsi_poll(struct cam_sim *sim)
2327 {
2328
2329         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2330 }
2331
2332 static void
2333 iscsi_terminate_sessions(struct iscsi_softc *sc)
2334 {
2335         struct iscsi_session *is;
2336
2337         sx_slock(&sc->sc_lock);
2338         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2339                 iscsi_session_terminate(is);
2340         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2341                 ISCSI_DEBUG("waiting for sessions to terminate");
2342                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2343         }
2344         ISCSI_DEBUG("all sessions terminated");
2345         sx_sunlock(&sc->sc_lock);
2346 }
2347
2348 static void
2349 iscsi_shutdown_pre(struct iscsi_softc *sc)
2350 {
2351         struct iscsi_session *is;
2352
2353         if (!fail_on_shutdown)
2354                 return;
2355
2356         /*
2357          * If we have any sessions waiting for reconnection, request
2358          * maintenance thread to fail them immediately instead of waiting
2359          * for reconnect timeout.
2360          *
2361          * This prevents LUNs with mounted filesystems that are supported
2362          * by disconnected iSCSI sessions from hanging, however it will
2363          * fail all queued BIOs.
2364          */
2365         ISCSI_DEBUG("forcing failing all disconnected sessions due to shutdown");
2366
2367         fail_on_disconnection = 1;
2368
2369         sx_slock(&sc->sc_lock);
2370         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
2371                 ISCSI_SESSION_LOCK(is);
2372                 if (!is->is_connected) {
2373                         ISCSI_SESSION_DEBUG(is, "force failing disconnected session early");
2374                         iscsi_session_reconnect(is);
2375                 }
2376                 ISCSI_SESSION_UNLOCK(is);
2377         }
2378         sx_sunlock(&sc->sc_lock);
2379 }
2380
2381 static void
2382 iscsi_shutdown_post(struct iscsi_softc *sc)
2383 {
2384
2385         ISCSI_DEBUG("removing all sessions due to shutdown");
2386         iscsi_terminate_sessions(sc);
2387 }
2388
2389 static int
2390 iscsi_load(void)
2391 {
2392         int error;
2393
2394         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2395         sx_init(&sc->sc_lock, "iscsi");
2396         TAILQ_INIT(&sc->sc_sessions);
2397         cv_init(&sc->sc_cv, "iscsi_cv");
2398
2399         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2400             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2401             UMA_ALIGN_PTR, 0);
2402
2403         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2404             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2405         if (error != 0) {
2406                 ISCSI_WARN("failed to create device node, error %d", error);
2407                 return (error);
2408         }
2409         sc->sc_cdev->si_drv1 = sc;
2410
2411         sc->sc_shutdown_pre_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
2412             iscsi_shutdown_pre, sc, SHUTDOWN_PRI_FIRST);
2413         /*
2414          * shutdown_post_sync needs to run after filesystem shutdown and before
2415          * CAM shutdown - otherwise when rebooting with an iSCSI session that is
2416          * disconnected but has outstanding requests, dashutdown() will hang on
2417          * cam_periph_runccb().
2418          */
2419         sc->sc_shutdown_post_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2420             iscsi_shutdown_post, sc, SHUTDOWN_PRI_DEFAULT - 1);
2421
2422         return (0);
2423 }
2424
2425 static int
2426 iscsi_unload(void)
2427 {
2428
2429         if (sc->sc_cdev != NULL) {
2430                 ISCSI_DEBUG("removing device node");
2431                 destroy_dev(sc->sc_cdev);
2432                 ISCSI_DEBUG("device node removed");
2433         }
2434
2435         if (sc->sc_shutdown_pre_eh != NULL)
2436                 EVENTHANDLER_DEREGISTER(shutdown_pre_sync, sc->sc_shutdown_pre_eh);
2437         if (sc->sc_shutdown_post_eh != NULL)
2438                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_post_eh);
2439
2440         iscsi_terminate_sessions(sc);
2441
2442         uma_zdestroy(iscsi_outstanding_zone);
2443         sx_destroy(&sc->sc_lock);
2444         cv_destroy(&sc->sc_cv);
2445         free(sc, M_ISCSI);
2446         return (0);
2447 }
2448
2449 static int
2450 iscsi_quiesce(void)
2451 {
2452         sx_slock(&sc->sc_lock);
2453         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2454                 sx_sunlock(&sc->sc_lock);
2455                 return (EBUSY);
2456         }
2457         sx_sunlock(&sc->sc_lock);
2458         return (0);
2459 }
2460
2461 static int
2462 iscsi_modevent(module_t mod, int what, void *arg)
2463 {
2464         int error;
2465
2466         switch (what) {
2467         case MOD_LOAD:
2468                 error = iscsi_load();
2469                 break;
2470         case MOD_UNLOAD:
2471                 error = iscsi_unload();
2472                 break;
2473         case MOD_QUIESCE:
2474                 error = iscsi_quiesce();
2475                 break;
2476         default:
2477                 error = EINVAL;
2478                 break;
2479         }
2480         return (error);
2481 }
2482
2483 moduledata_t iscsi_data = {
2484         "iscsi",
2485         iscsi_modevent,
2486         0
2487 };
2488
2489 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2490 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2491 MODULE_DEPEND(iscsi, icl, 1, 1, 1);