sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/param.h>
  35 #include <sys/condvar.h>
  36 #include <sys/conf.h>
  37 #include <sys/endian.h>
  38 #include <sys/eventhandler.h>
  39 #include <sys/file.h>
  40 #include <sys/kernel.h>
  41 #include <sys/kthread.h>
  42 #include <sys/lock.h>
  43 #include <sys/malloc.h>
  44 #include <sys/mutex.h>
  45 #include <sys/module.h>
  46 #include <sys/sysctl.h>
  47 #include <sys/systm.h>
  48 #include <sys/sx.h>
  49 #include <vm/uma.h>
  50
  51 #include <cam/cam.h>
  52 #include <cam/cam_ccb.h>
  53 #include <cam/cam_xpt.h>
  54 #include <cam/cam_debug.h>
  55 #include <cam/cam_sim.h>
  56 #include <cam/cam_xpt_sim.h>
  57 #include <cam/cam_xpt_periph.h>
  58 #include <cam/cam_periph.h>
  59 #include <cam/scsi/scsi_all.h>
  60 #include <cam/scsi/scsi_message.h>
  61
  62 #include <dev/iscsi/icl.h>
  63 #include <dev/iscsi/icl_wrappers.h>
  64 #include <dev/iscsi/iscsi_ioctl.h>
  65 #include <dev/iscsi/iscsi_proto.h>
  66 #include <dev/iscsi/iscsi.h>
  67
  68 #ifdef ICL_KERNEL_PROXY
  69 #include <sys/socketvar.h>
  70 #endif
  71
  72 #ifdef ICL_KERNEL_PROXY
  73 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  74 #endif
  75
  76 /*
  77  * XXX: This is global so the iscsi_unload() can access it.
  78  *      Think about how to do this properly.
  79  */
  80 static struct iscsi_softc       *sc;
  81
  82 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  83 static int debug = 1;
  84 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  85     &debug, 0, "Enable debug messages");
  86 static int ping_timeout = 5;
  87 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  88     0, "Timeout for ping (NOP-Out) requests, in seconds");
  89 static int iscsid_timeout = 60;
  90 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  91     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  92 static int login_timeout = 60;
  93 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
  94     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  95 static int maxtags = 255;
  96 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
  97     0, "Max number of IO requests queued");
  98 static int fail_on_disconnection = 0;
  99 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
 100     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
 101
 102 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 103 static uma_zone_t iscsi_outstanding_zone;
 104
 105 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 106 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 107
 108 #define ISCSI_DEBUG(X, ...)                                             \
 109         do {                                                            \
 110                 if (debug > 1)                                          \
 111                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 112         } while (0)
 113
 114 #define ISCSI_WARN(X, ...)                                              \
 115         do {                                                            \
 116                 if (debug > 0) {                                        \
 117                         printf("WARNING: %s: " X "\n",                  \
 118                             __func__, ## __VA_ARGS__);                  \
 119                 }                                                       \
 120         } while (0)
 121
 122 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 123         do {                                                            \
 124                 if (debug > 1) {                                        \
 125                         printf("%s: %s (%s): " X "\n",                  \
 126                             __func__, S->is_conf.isc_target_addr,       \
 127                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 128                 }                                                       \
 129         } while (0)
 130
 131 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 132         do {                                                            \
 133                 if (debug > 0) {                                        \
 134                         printf("WARNING: %s (%s): " X "\n",             \
 135                             S->is_conf.isc_target_addr,                 \
 136                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 137                 }                                                       \
 138         } while (0)
 139
 140 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 141 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 142 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 143 #define ISCSI_SESSION_LOCK_ASSERT_NOT(X) mtx_assert(&X->is_lock, MA_NOTOWNED)
 144
 145 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 146                     int mode, struct thread *td);
 147
 148 static struct cdevsw iscsi_cdevsw = {
 149      .d_version = D_VERSION,
 150      .d_ioctl   = iscsi_ioctl,
 151      .d_name    = "iscsi",
 152 };
 153
 154 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 155 static void     iscsi_pdu_queue(struct icl_pdu *request);
 156 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 157 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 158 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 159 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
 160 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 161 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 162 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 163 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 164 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 165 static void     iscsi_session_reconnect(struct iscsi_session *is);
 166 static void     iscsi_session_terminate(struct iscsi_session *is);
 167 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 168 static void     iscsi_poll(struct cam_sim *sim);
 169 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 170                     uint32_t initiator_task_tag);
 171 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
 172                     union ccb *ccb, uint32_t *initiator_task_tagp);
 173 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 174                     struct iscsi_outstanding *io);
 175
 176 static bool
 177 iscsi_pdu_prepare(struct icl_pdu *request)
 178 {
 179         struct iscsi_session *is;
 180         struct iscsi_bhs_scsi_command *bhssc;
 181
 182         is = PDU_SESSION(request);
 183
 184         ISCSI_SESSION_LOCK_ASSERT(is);
 185
 186         /*
 187          * We're only using fields common for all the request
 188          * (initiator -> target) PDUs.
 189          */
 190         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 191
 192         /*
 193          * Data-Out PDU does not contain CmdSN.
 194          */
 195         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 196                 if (ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn) &&
 197                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 198                         /*
 199                          * Current MaxCmdSN prevents us from sending any more
 200                          * SCSI Command PDUs to the target; postpone the PDU.
 201                          * It will get resent by either iscsi_pdu_queue(),
 202                          * or by maintenance thread.
 203                          */
 204 #if 0
 205                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %u, "
 206                             "ExpCmdSN %u, MaxCmdSN %u, opcode 0x%x",
 207                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn,
 208                             bhssc->bhssc_opcode);
 209 #endif
 210                         return (true);
 211                 }
 212                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 213                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 214                         is->is_cmdsn++;
 215         }
 216         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 217
 218         return (false);
 219 }
 220
 221 static void
 222 iscsi_session_send_postponed(struct iscsi_session *is)
 223 {
 224         struct icl_pdu *request;
 225         bool postpone;
 226
 227         ISCSI_SESSION_LOCK_ASSERT(is);
 228
 229         while (!STAILQ_EMPTY(&is->is_postponed)) {
 230                 request = STAILQ_FIRST(&is->is_postponed);
 231                 postpone = iscsi_pdu_prepare(request);
 232                 if (postpone)
 233                         break;
 234                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 235                 icl_pdu_queue(request);
 236         }
 237 }
 238
 239 static void
 240 iscsi_pdu_queue_locked(struct icl_pdu *request)
 241 {
 242         struct iscsi_session *is;
 243         bool postpone;
 244
 245         is = PDU_SESSION(request);
 246         ISCSI_SESSION_LOCK_ASSERT(is);
 247         iscsi_session_send_postponed(is);
 248         postpone = iscsi_pdu_prepare(request);
 249         if (postpone) {
 250                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 251                 return;
 252         }
 253         icl_pdu_queue(request);
 254 }
 255
 256 static void
 257 iscsi_pdu_queue(struct icl_pdu *request)
 258 {
 259         struct iscsi_session *is;
 260
 261         is = PDU_SESSION(request);
 262         ISCSI_SESSION_LOCK(is);
 263         iscsi_pdu_queue_locked(request);
 264         ISCSI_SESSION_UNLOCK(is);
 265 }
 266
 267 static void
 268 iscsi_session_logout(struct iscsi_session *is)
 269 {
 270         struct icl_pdu *request;
 271         struct iscsi_bhs_logout_request *bhslr;
 272
 273         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 274         if (request == NULL)
 275                 return;
 276
 277         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 278         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 279         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 280         iscsi_pdu_queue_locked(request);
 281 }
 282
 283 static void
 284 iscsi_session_terminate_task(struct iscsi_session *is,
 285     struct iscsi_outstanding *io, bool requeue)
 286 {
 287
 288         if (io->io_ccb != NULL) {
 289                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
 290                 if (requeue)
 291                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 292                 else
 293                         io->io_ccb->ccb_h.status |= CAM_REQ_ABORTED;
 294                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 295                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 296                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 297                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 298                 }
 299                 xpt_done(io->io_ccb);
 300         }
 301         iscsi_outstanding_remove(is, io);
 302 }
 303
 304 static void
 305 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 306 {
 307         struct iscsi_outstanding *io, *tmp;
 308
 309         ISCSI_SESSION_LOCK_ASSERT(is);
 310
 311         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 312                 iscsi_session_terminate_task(is, io, requeue);
 313         }
 314 }
 315
 316 static void
 317 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 318 {
 319         struct icl_pdu *pdu;
 320
 321         ISCSI_SESSION_LOCK_ASSERT(is);
 322
 323         /*
 324          * Don't queue any new PDUs.
 325          */
 326         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 327                 ISCSI_SESSION_DEBUG(is, "freezing");
 328                 xpt_freeze_simq(is->is_sim, 1);
 329                 is->is_simq_frozen = true;
 330         }
 331
 332         /*
 333          * Remove postponed PDUs.
 334          */
 335         while (!STAILQ_EMPTY(&is->is_postponed)) {
 336                 pdu = STAILQ_FIRST(&is->is_postponed);
 337                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 338                 icl_pdu_free(pdu);
 339         }
 340
 341         if (destroy_sim == false) {
 342                 /*
 343                  * Terminate SCSI tasks, asking CAM to requeue them.
 344                  */
 345                 iscsi_session_terminate_tasks(is, true);
 346                 return;
 347         }
 348
 349         iscsi_session_terminate_tasks(is, false);
 350
 351         if (is->is_sim == NULL)
 352                 return;
 353
 354         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 355         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 356
 357         if (is->is_simq_frozen) {
 358                 xpt_release_simq(is->is_sim, 1);
 359                 is->is_simq_frozen = false;
 360         }
 361
 362         xpt_free_path(is->is_path);
 363         is->is_path = NULL;
 364         xpt_bus_deregister(cam_sim_path(is->is_sim));
 365         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 366         is->is_sim = NULL;
 367         is->is_devq = NULL;
 368 }
 369
 370 static void
 371 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 372 {
 373
 374         icl_conn_close(is->is_conn);
 375
 376         ISCSI_SESSION_LOCK(is);
 377
 378         is->is_connected = false;
 379         is->is_reconnecting = false;
 380         is->is_login_phase = false;
 381
 382 #ifdef ICL_KERNEL_PROXY
 383         if (is->is_login_pdu != NULL) {
 384                 icl_pdu_free(is->is_login_pdu);
 385                 is->is_login_pdu = NULL;
 386         }
 387         cv_signal(&is->is_login_cv);
 388 #endif
 389
 390         if (fail_on_disconnection) {
 391                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
 392                 iscsi_session_cleanup(is, true);
 393         } else {
 394                 iscsi_session_cleanup(is, false);
 395         }
 396
 397         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 398             ("destroying session with active tasks"));
 399         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 400             ("destroying session with postponed PDUs"));
 401
 402         /*
 403          * Request immediate reconnection from iscsid(8).
 404          */
 405         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 406         is->is_waiting_for_iscsid = true;
 407         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 408         is->is_timeout = 0;
 409         ISCSI_SESSION_UNLOCK(is);
 410         cv_signal(&is->is_softc->sc_cv);
 411 }
 412
 413 static void
 414 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 415 {
 416         struct iscsi_softc *sc;
 417
 418         sc = is->is_softc;
 419         sx_xlock(&sc->sc_lock);
 420         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 421         sx_xunlock(&sc->sc_lock);
 422
 423         icl_conn_close(is->is_conn);
 424         callout_drain(&is->is_callout);
 425
 426         ISCSI_SESSION_LOCK(is);
 427
 428         KASSERT(is->is_terminating, ("is_terminating == false"));
 429
 430 #ifdef ICL_KERNEL_PROXY
 431         if (is->is_login_pdu != NULL) {
 432                 icl_pdu_free(is->is_login_pdu);
 433                 is->is_login_pdu = NULL;
 434         }
 435         cv_signal(&is->is_login_cv);
 436 #endif
 437
 438         iscsi_session_cleanup(is, true);
 439
 440         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 441             ("destroying session with active tasks"));
 442         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 443             ("destroying session with postponed PDUs"));
 444
 445         ISCSI_SESSION_UNLOCK(is);
 446
 447         icl_conn_free(is->is_conn);
 448         mtx_destroy(&is->is_lock);
 449         cv_destroy(&is->is_maintenance_cv);
 450 #ifdef ICL_KERNEL_PROXY
 451         cv_destroy(&is->is_login_cv);
 452 #endif
 453         ISCSI_SESSION_DEBUG(is, "terminated");
 454         free(is, M_ISCSI);
 455
 456         /*
 457          * The iscsi_unload() routine might be waiting.
 458          */
 459         cv_signal(&sc->sc_cv);
 460 }
 461
 462 static void
 463 iscsi_maintenance_thread(void *arg)
 464 {
 465         struct iscsi_session *is;
 466
 467         is = arg;
 468
 469         for (;;) {
 470                 ISCSI_SESSION_LOCK(is);
 471                 if (is->is_reconnecting == false &&
 472                     is->is_terminating == false &&
 473                     STAILQ_EMPTY(&is->is_postponed))
 474                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 475
 476                 if (is->is_reconnecting) {
 477                         ISCSI_SESSION_UNLOCK(is);
 478                         iscsi_maintenance_thread_reconnect(is);
 479                         continue;
 480                 }
 481
 482                 if (is->is_terminating) {
 483                         ISCSI_SESSION_UNLOCK(is);
 484                         iscsi_maintenance_thread_terminate(is);
 485                         kthread_exit();
 486                         return;
 487                 }
 488
 489                 iscsi_session_send_postponed(is);
 490                 ISCSI_SESSION_UNLOCK(is);
 491         }
 492 }
 493
 494 static void
 495 iscsi_session_reconnect(struct iscsi_session *is)
 496 {
 497
 498         /*
 499          * XXX: We can't use locking here, because
 500          *      it's being called from various contexts.
 501          *      Hope it doesn't break anything.
 502          */
 503         if (is->is_reconnecting)
 504                 return;
 505
 506         is->is_reconnecting = true;
 507         cv_signal(&is->is_maintenance_cv);
 508 }
 509
 510 static void
 511 iscsi_session_terminate(struct iscsi_session *is)
 512 {
 513
 514         if (is->is_terminating)
 515                 return;
 516
 517         is->is_terminating = true;
 518
 519 #if 0
 520         iscsi_session_logout(is);
 521 #endif
 522         cv_signal(&is->is_maintenance_cv);
 523 }
 524
 525 static void
 526 iscsi_callout(void *context)
 527 {
 528         struct icl_pdu *request;
 529         struct iscsi_bhs_nop_out *bhsno;
 530         struct iscsi_session *is;
 531         bool reconnect_needed = false;
 532
 533         is = context;
 534
 535         ISCSI_SESSION_LOCK(is);
 536         if (is->is_terminating) {
 537                 ISCSI_SESSION_UNLOCK(is);
 538                 return;
 539         }
 540
 541         callout_schedule(&is->is_callout, 1 * hz);
 542
 543         is->is_timeout++;
 544
 545         if (is->is_waiting_for_iscsid) {
 546                 if (iscsid_timeout > 0 && is->is_timeout > iscsid_timeout) {
 547                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 548                             "for %d seconds; reconnecting",
 549                             is->is_timeout);
 550                         reconnect_needed = true;
 551                 }
 552                 goto out;
 553         }
 554
 555         if (is->is_login_phase) {
 556                 if (login_timeout > 0 && is->is_timeout > login_timeout) {
 557                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 558                             "reconnecting", is->is_timeout);
 559                         reconnect_needed = true;
 560                 }
 561                 goto out;
 562         }
 563
 564         if (ping_timeout <= 0) {
 565                 /*
 566                  * Pings are disabled.  Don't send NOP-Out in this case.
 567                  * Reset the timeout, to avoid triggering reconnection,
 568                  * should the user decide to reenable them.
 569                  */
 570                 is->is_timeout = 0;
 571                 goto out;
 572         }
 573
 574         if (is->is_timeout >= ping_timeout) {
 575                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 576                     "reconnecting", ping_timeout);
 577                 reconnect_needed = true;
 578                 goto out;
 579         }
 580
 581         ISCSI_SESSION_UNLOCK(is);
 582
 583         /*
 584          * If the ping was reset less than one second ago - which means
 585          * that we've received some PDU during the last second - assume
 586          * the traffic flows correctly and don't bother sending a NOP-Out.
 587          *
 588          * (It's 2 - one for one second, and one for incrementing is_timeout
 589          * earlier in this routine.)
 590          */
 591         if (is->is_timeout < 2)
 592                 return;
 593
 594         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 595         if (request == NULL) {
 596                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 597                 return;
 598         }
 599         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 600         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 601             ISCSI_BHS_OPCODE_IMMEDIATE;
 602         bhsno->bhsno_flags = 0x80;
 603         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 604         iscsi_pdu_queue(request);
 605         return;
 606
 607 out:
 608         ISCSI_SESSION_UNLOCK(is);
 609
 610         if (reconnect_needed)
 611                 iscsi_session_reconnect(is);
 612 }
 613
 614 static void
 615 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 616 {
 617         const struct iscsi_bhs_data_in *bhsdi;
 618         struct iscsi_session *is;
 619         uint32_t expcmdsn, maxcmdsn, statsn;
 620
 621         is = PDU_SESSION(response);
 622
 623         ISCSI_SESSION_LOCK_ASSERT(is);
 624
 625         /*
 626          * We're only using fields common for all the response
 627          * (target -> initiator) PDUs.
 628          */
 629         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 630         /*
 631          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 632          * and Residual Count only have meaningful content if the S bit
 633          * is set to 1", so we also need to check the bit specific for
 634          * Data-In PDU.
 635          */
 636         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 637             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 638                 statsn = ntohl(bhsdi->bhsdi_statsn);
 639                 if (statsn != is->is_statsn && statsn != (is->is_statsn + 1)) {
 640                         /* XXX: This is normal situation for MCS */
 641                         ISCSI_SESSION_WARN(is, "PDU 0x%x StatSN %u != "
 642                             "session ExpStatSN %u (or + 1); reconnecting",
 643                             bhsdi->bhsdi_opcode, statsn, is->is_statsn);
 644                         iscsi_session_reconnect(is);
 645                 }
 646                 if (ISCSI_SNGT(statsn, is->is_statsn))
 647                         is->is_statsn = statsn;
 648         }
 649
 650         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 651         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 652
 653         if (ISCSI_SNLT(maxcmdsn + 1, expcmdsn)) {
 654                 ISCSI_SESSION_DEBUG(is,
 655                     "PDU MaxCmdSN %u + 1 < PDU ExpCmdSN %u; ignoring",
 656                     maxcmdsn, expcmdsn);
 657         } else {
 658                 if (ISCSI_SNGT(maxcmdsn, is->is_maxcmdsn)) {
 659                         is->is_maxcmdsn = maxcmdsn;
 660
 661                         /*
 662                          * Command window increased; kick the maintanance thread
 663                          * to send out postponed commands.
 664                          */
 665                         if (!STAILQ_EMPTY(&is->is_postponed))
 666                                 cv_signal(&is->is_maintenance_cv);
 667                 } else if (ISCSI_SNLT(maxcmdsn, is->is_maxcmdsn)) {
 668                         /* XXX: This is normal situation for MCS */
 669                         ISCSI_SESSION_DEBUG(is,
 670                             "PDU MaxCmdSN %u < session MaxCmdSN %u; ignoring",
 671                             maxcmdsn, is->is_maxcmdsn);
 672                 }
 673
 674                 if (ISCSI_SNGT(expcmdsn, is->is_expcmdsn)) {
 675                         is->is_expcmdsn = expcmdsn;
 676                 } else if (ISCSI_SNLT(expcmdsn, is->is_expcmdsn)) {
 677                         /* XXX: This is normal situation for MCS */
 678                         ISCSI_SESSION_DEBUG(is,
 679                             "PDU ExpCmdSN %u < session ExpCmdSN %u; ignoring",
 680                             expcmdsn, is->is_expcmdsn);
 681                 }
 682         }
 683
 684         /*
 685          * Every incoming PDU - not just NOP-In - resets the ping timer.
 686          * The purpose of the timeout is to reset the connection when it stalls;
 687          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 688          * in some queue.
 689          */
 690         is->is_timeout = 0;
 691 }
 692
 693 static void
 694 iscsi_receive_callback(struct icl_pdu *response)
 695 {
 696         struct iscsi_session *is;
 697
 698         is = PDU_SESSION(response);
 699
 700         ISCSI_SESSION_LOCK(is);
 701
 702 #ifdef ICL_KERNEL_PROXY
 703         if (is->is_login_phase) {
 704                 if (is->is_login_pdu == NULL)
 705                         is->is_login_pdu = response;
 706                 else
 707                         icl_pdu_free(response);
 708                 ISCSI_SESSION_UNLOCK(is);
 709                 cv_signal(&is->is_login_cv);
 710                 return;
 711         }
 712 #endif
 713
 714         iscsi_pdu_update_statsn(response);
 715
 716         /*
 717          * The handling routine is responsible for freeing the PDU
 718          * when it's no longer needed.
 719          */
 720         switch (response->ip_bhs->bhs_opcode) {
 721         case ISCSI_BHS_OPCODE_NOP_IN:
 722                 iscsi_pdu_handle_nop_in(response);
 723                 ISCSI_SESSION_UNLOCK(is);
 724                 break;
 725         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 726                 iscsi_pdu_handle_scsi_response(response);
 727                 /* Session lock dropped inside. */
 728                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 729                 break;
 730         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
 731                 iscsi_pdu_handle_task_response(response);
 732                 ISCSI_SESSION_UNLOCK(is);
 733                 break;
 734         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 735                 iscsi_pdu_handle_data_in(response);
 736                 /* Session lock dropped inside. */
 737                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 738                 break;
 739         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 740                 iscsi_pdu_handle_logout_response(response);
 741                 ISCSI_SESSION_UNLOCK(is);
 742                 break;
 743         case ISCSI_BHS_OPCODE_R2T:
 744                 iscsi_pdu_handle_r2t(response);
 745                 ISCSI_SESSION_UNLOCK(is);
 746                 break;
 747         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 748                 iscsi_pdu_handle_async_message(response);
 749                 ISCSI_SESSION_UNLOCK(is);
 750                 break;
 751         case ISCSI_BHS_OPCODE_REJECT:
 752                 iscsi_pdu_handle_reject(response);
 753                 ISCSI_SESSION_UNLOCK(is);
 754                 break;
 755         default:
 756                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 757                     "opcode 0x%x; reconnecting",
 758                     response->ip_bhs->bhs_opcode);
 759                 iscsi_session_reconnect(is);
 760                 ISCSI_SESSION_UNLOCK(is);
 761                 icl_pdu_free(response);
 762         }
 763 }
 764
 765 static void
 766 iscsi_error_callback(struct icl_conn *ic)
 767 {
 768         struct iscsi_session *is;
 769
 770         is = CONN_SESSION(ic);
 771
 772         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 773         iscsi_session_reconnect(is);
 774 }
 775
 776 static void
 777 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 778 {
 779         struct iscsi_session *is;
 780         struct iscsi_bhs_nop_out *bhsno;
 781         struct iscsi_bhs_nop_in *bhsni;
 782         struct icl_pdu *request;
 783         void *data = NULL;
 784         size_t datasize;
 785         int error;
 786
 787         is = PDU_SESSION(response);
 788         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 789
 790         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 791                 /*
 792                  * Nothing to do; iscsi_pdu_update_statsn() already
 793                  * zeroed the timeout.
 794                  */
 795                 icl_pdu_free(response);
 796                 return;
 797         }
 798
 799         datasize = icl_pdu_data_segment_length(response);
 800         if (datasize > 0) {
 801                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 802                 if (data == NULL) {
 803                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 804                             "reconnecting");
 805                         icl_pdu_free(response);
 806                         iscsi_session_reconnect(is);
 807                         return;
 808                 }
 809                 icl_pdu_get_data(response, 0, data, datasize);
 810         }
 811
 812         request = icl_pdu_new(response->ip_conn, M_NOWAIT);
 813         if (request == NULL) {
 814                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 815                     "reconnecting");
 816                 free(data, M_ISCSI);
 817                 icl_pdu_free(response);
 818                 iscsi_session_reconnect(is);
 819                 return;
 820         }
 821         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 822         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 823             ISCSI_BHS_OPCODE_IMMEDIATE;
 824         bhsno->bhsno_flags = 0x80;
 825         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 826         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 827         if (datasize > 0) {
 828                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 829                 if (error != 0) {
 830                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 831                             "reconnecting");
 832                         free(data, M_ISCSI);
 833                         icl_pdu_free(request);
 834                         icl_pdu_free(response);
 835                         iscsi_session_reconnect(is);
 836                         return;
 837                 }
 838                 free(data, M_ISCSI);
 839         }
 840
 841         icl_pdu_free(response);
 842         iscsi_pdu_queue_locked(request);
 843 }
 844
 845 static void
 846 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 847 {
 848         struct iscsi_bhs_scsi_response *bhssr;
 849         struct iscsi_outstanding *io;
 850         struct iscsi_session *is;
 851         union ccb *ccb;
 852         struct ccb_scsiio *csio;
 853         size_t data_segment_len, received;
 854         uint16_t sense_len;
 855
 856         is = PDU_SESSION(response);
 857
 858         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 859         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 860         if (io == NULL || io->io_ccb == NULL) {
 861                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 862                 icl_pdu_free(response);
 863                 iscsi_session_reconnect(is);
 864                 ISCSI_SESSION_UNLOCK(is);
 865                 return;
 866         }
 867
 868         ccb = io->io_ccb;
 869         received = io->io_received;
 870         iscsi_outstanding_remove(is, io);
 871         ISCSI_SESSION_UNLOCK(is);
 872
 873         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 874                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 875                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 876                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 877                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 878                 }
 879                 ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 880         } else if (bhssr->bhssr_status == 0) {
 881                 ccb->ccb_h.status = CAM_REQ_CMP;
 882         } else {
 883                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 884                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 885                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 886                 }
 887                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 888                 ccb->csio.scsi_status = bhssr->bhssr_status;
 889         }
 890
 891         csio = &ccb->csio;
 892         data_segment_len = icl_pdu_data_segment_length(response);
 893         if (data_segment_len > 0) {
 894                 if (data_segment_len < sizeof(sense_len)) {
 895                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 896                             data_segment_len);
 897                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 898                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 899                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 900                         }
 901                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 902                         goto out;
 903                 }
 904                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 905                 sense_len = ntohs(sense_len);
 906 #if 0
 907                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 908                     sense_len, data_segment_len);
 909 #endif
 910                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 911                         ISCSI_SESSION_WARN(is, "truncated data segment "
 912                             "(%zd bytes, should be %zd)",
 913                             data_segment_len, sizeof(sense_len) + sense_len);
 914                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 915                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
 916                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 917                         }
 918                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 919                         goto out;
 920                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 921                         ISCSI_SESSION_WARN(is, "oversize data segment "
 922                             "(%zd bytes, should be %zd)",
 923                             data_segment_len, sizeof(sense_len) + sense_len);
 924                 if (sense_len > csio->sense_len) {
 925                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 926                             sense_len, csio->sense_len);
 927                         sense_len = csio->sense_len;
 928                 }
 929                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 930                 csio->sense_resid = csio->sense_len - sense_len;
 931                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 932         }
 933
 934 out:
 935         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 936                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 937
 938         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 939                 KASSERT(received <= csio->dxfer_len,
 940                     ("received > csio->dxfer_len"));
 941                 if (received < csio->dxfer_len) {
 942                         if (csio->resid != csio->dxfer_len - received) {
 943                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 944                                     "target indicates %d, we calculated %zd",
 945                                     csio->resid, csio->dxfer_len - received);
 946                         }
 947                         csio->resid = csio->dxfer_len - received;
 948                 }
 949         }
 950
 951         xpt_done(ccb);
 952         icl_pdu_free(response);
 953 }
 954
 955 static void
 956 iscsi_pdu_handle_task_response(struct icl_pdu *response)
 957 {
 958         struct iscsi_bhs_task_management_response *bhstmr;
 959         struct iscsi_outstanding *io, *aio;
 960         struct iscsi_session *is;
 961
 962         is = PDU_SESSION(response);
 963
 964         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
 965         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
 966         if (io == NULL || io->io_ccb != NULL) {
 967                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
 968                     bhstmr->bhstmr_initiator_task_tag);
 969                 icl_pdu_free(response);
 970                 iscsi_session_reconnect(is);
 971                 return;
 972         }
 973
 974         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
 975                 ISCSI_SESSION_WARN(is, "task response 0x%x",
 976                     bhstmr->bhstmr_response);
 977         } else {
 978                 aio = iscsi_outstanding_find(is, io->io_datasn);
 979                 if (aio != NULL && aio->io_ccb != NULL)
 980                         iscsi_session_terminate_task(is, aio, false);
 981         }
 982
 983         iscsi_outstanding_remove(is, io);
 984         icl_pdu_free(response);
 985 }
 986
 987 static void
 988 iscsi_pdu_handle_data_in(struct icl_pdu *response)
 989 {
 990         struct iscsi_bhs_data_in *bhsdi;
 991         struct iscsi_outstanding *io;
 992         struct iscsi_session *is;
 993         union ccb *ccb;
 994         struct ccb_scsiio *csio;
 995         size_t data_segment_len, received, oreceived;
 996
 997         is = PDU_SESSION(response);
 998         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
 999         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
1000         if (io == NULL || io->io_ccb == NULL) {
1001                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
1002                 icl_pdu_free(response);
1003                 iscsi_session_reconnect(is);
1004                 ISCSI_SESSION_UNLOCK(is);
1005                 return;
1006         }
1007
1008         data_segment_len = icl_pdu_data_segment_length(response);
1009         if (data_segment_len == 0) {
1010                 /*
1011                  * "The sending of 0 length data segments should be avoided,
1012                  * but initiators and targets MUST be able to properly receive
1013                  * 0 length data segments."
1014                  */
1015                 ISCSI_SESSION_UNLOCK(is);
1016                 icl_pdu_free(response);
1017                 return;
1018         }
1019
1020         /*
1021          * We need to track this for security reasons - without it, malicious target
1022          * could respond to SCSI READ without sending Data-In PDUs, which would result
1023          * in read operation on the initiator side returning random kernel data.
1024          */
1025         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1026                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1027                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1028                 icl_pdu_free(response);
1029                 iscsi_session_reconnect(is);
1030                 ISCSI_SESSION_UNLOCK(is);
1031                 return;
1032         }
1033
1034         ccb = io->io_ccb;
1035         csio = &ccb->csio;
1036
1037         if (io->io_received + data_segment_len > csio->dxfer_len) {
1038                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1039                     "at offset %zd, buffer is %d)",
1040                     data_segment_len, io->io_received, csio->dxfer_len);
1041                 icl_pdu_free(response);
1042                 iscsi_session_reconnect(is);
1043                 ISCSI_SESSION_UNLOCK(is);
1044                 return;
1045         }
1046
1047         oreceived = io->io_received;
1048         io->io_received += data_segment_len;
1049         received = io->io_received;
1050         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0)
1051                 iscsi_outstanding_remove(is, io);
1052         ISCSI_SESSION_UNLOCK(is);
1053
1054         icl_pdu_get_data(response, 0, csio->data_ptr + oreceived, data_segment_len);
1055
1056         /*
1057          * XXX: Check DataSN.
1058          * XXX: Check F.
1059          */
1060         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1061                 /*
1062                  * Nothing more to do.
1063                  */
1064                 icl_pdu_free(response);
1065                 return;
1066         }
1067
1068         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1069         if (bhsdi->bhsdi_status == 0) {
1070                 ccb->ccb_h.status = CAM_REQ_CMP;
1071         } else {
1072                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1073                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1074                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1075                 }
1076                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1077                 csio->scsi_status = bhsdi->bhsdi_status;
1078         }
1079
1080         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1081                 KASSERT(received <= csio->dxfer_len,
1082                     ("received > csio->dxfer_len"));
1083                 if (received < csio->dxfer_len) {
1084                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1085                         if (csio->resid != csio->dxfer_len - received) {
1086                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1087                                     "target indicates %d, we calculated %zd",
1088                                     csio->resid, csio->dxfer_len - received);
1089                         }
1090                         csio->resid = csio->dxfer_len - received;
1091                 }
1092         }
1093
1094         xpt_done(ccb);
1095         icl_pdu_free(response);
1096 }
1097
1098 static void
1099 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1100 {
1101
1102         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1103         icl_pdu_free(response);
1104 }
1105
1106 static void
1107 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1108 {
1109         struct icl_pdu *request;
1110         struct iscsi_session *is;
1111         struct iscsi_bhs_r2t *bhsr2t;
1112         struct iscsi_bhs_data_out *bhsdo;
1113         struct iscsi_outstanding *io;
1114         struct ccb_scsiio *csio;
1115         size_t off, len, total_len;
1116         int error;
1117
1118         is = PDU_SESSION(response);
1119
1120         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1121         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1122         if (io == NULL || io->io_ccb == NULL) {
1123                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1124                     bhsr2t->bhsr2t_initiator_task_tag);
1125                 icl_pdu_free(response);
1126                 iscsi_session_reconnect(is);
1127                 return;
1128         }
1129
1130         csio = &io->io_ccb->csio;
1131
1132         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1133                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1134                 icl_pdu_free(response);
1135                 iscsi_session_reconnect(is);
1136                 return;
1137         }
1138
1139         /*
1140          * XXX: Verify R2TSN.
1141          */
1142
1143         io->io_datasn = 0;
1144
1145         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1146         if (off > csio->dxfer_len) {
1147                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1148                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1149                 icl_pdu_free(response);
1150                 iscsi_session_reconnect(is);
1151                 return;
1152         }
1153
1154         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1155         if (total_len == 0 || total_len > csio->dxfer_len) {
1156                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1157                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1158                 icl_pdu_free(response);
1159                 iscsi_session_reconnect(is);
1160                 return;
1161         }
1162
1163         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1164
1165         for (;;) {
1166                 len = total_len;
1167
1168                 if (len > is->is_max_data_segment_length)
1169                         len = is->is_max_data_segment_length;
1170
1171                 if (off + len > csio->dxfer_len) {
1172                         ISCSI_SESSION_WARN(is, "target requested invalid "
1173                             "length/offset %zd, buffer is %d; reconnecting",
1174                             off + len, csio->dxfer_len);
1175                         icl_pdu_free(response);
1176                         iscsi_session_reconnect(is);
1177                         return;
1178                 }
1179
1180                 request = icl_pdu_new(response->ip_conn, M_NOWAIT);
1181                 if (request == NULL) {
1182                         icl_pdu_free(response);
1183                         iscsi_session_reconnect(is);
1184                         return;
1185                 }
1186
1187                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1188                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1189                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1190                 bhsdo->bhsdo_initiator_task_tag =
1191                     bhsr2t->bhsr2t_initiator_task_tag;
1192                 bhsdo->bhsdo_target_transfer_tag =
1193                     bhsr2t->bhsr2t_target_transfer_tag;
1194                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1195                 bhsdo->bhsdo_buffer_offset = htonl(off);
1196                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1197                     M_NOWAIT);
1198                 if (error != 0) {
1199                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1200                             "reconnecting");
1201                         icl_pdu_free(request);
1202                         icl_pdu_free(response);
1203                         iscsi_session_reconnect(is);
1204                         return;
1205                 }
1206
1207                 off += len;
1208                 total_len -= len;
1209
1210                 if (total_len == 0) {
1211                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1212                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1213                 } else {
1214                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1215                 }
1216
1217                 iscsi_pdu_queue_locked(request);
1218
1219                 if (total_len == 0)
1220                         break;
1221         }
1222
1223         icl_pdu_free(response);
1224 }
1225
1226 static void
1227 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1228 {
1229         struct iscsi_bhs_asynchronous_message *bhsam;
1230         struct iscsi_session *is;
1231
1232         is = PDU_SESSION(response);
1233         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1234         switch (bhsam->bhsam_async_event) {
1235         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1236                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1237                 iscsi_session_logout(is);
1238                 iscsi_session_terminate(is);
1239                 break;
1240         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1241                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1242                 break;
1243         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1244                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1245                 break;
1246         default:
1247                 /*
1248                  * XXX: Technically, we're obligated to also handle
1249                  *      parameter renegotiation.
1250                  */
1251                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1252                 break;
1253         }
1254
1255         icl_pdu_free(response);
1256 }
1257
1258 static void
1259 iscsi_pdu_handle_reject(struct icl_pdu *response)
1260 {
1261         struct iscsi_bhs_reject *bhsr;
1262         struct iscsi_session *is;
1263
1264         is = PDU_SESSION(response);
1265         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1266         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1267             bhsr->bhsr_reason);
1268
1269         icl_pdu_free(response);
1270 }
1271
1272 static int
1273 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1274     struct iscsi_daemon_request *request)
1275 {
1276         struct iscsi_session *is;
1277         int error;
1278
1279         sx_slock(&sc->sc_lock);
1280         for (;;) {
1281                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1282                         ISCSI_SESSION_LOCK(is);
1283                         if (is->is_waiting_for_iscsid)
1284                                 break;
1285                         ISCSI_SESSION_UNLOCK(is);
1286                 }
1287
1288                 if (is == NULL) {
1289                         /*
1290                          * No session requires attention from iscsid(8); wait.
1291                          */
1292                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1293                         if (error != 0) {
1294                                 sx_sunlock(&sc->sc_lock);
1295                                 return (error);
1296                         }
1297                         continue;
1298                 }
1299
1300                 is->is_waiting_for_iscsid = false;
1301                 is->is_login_phase = true;
1302                 is->is_reason[0] = '\0';
1303                 ISCSI_SESSION_UNLOCK(is);
1304
1305                 request->idr_session_id = is->is_id;
1306                 memcpy(&request->idr_isid, &is->is_isid,
1307                     sizeof(request->idr_isid));
1308                 request->idr_tsih = 0;  /* New or reinstated session. */
1309                 memcpy(&request->idr_conf, &is->is_conf,
1310                     sizeof(request->idr_conf));
1311
1312                 error = icl_limits(is->is_conf.isc_offload,
1313                     &request->idr_limits.isl_max_data_segment_length);
1314                 if (error != 0) {
1315                         ISCSI_SESSION_WARN(is, "icl_limits for offload \"%s\" "
1316                             "failed with error %d", is->is_conf.isc_offload,
1317                             error);
1318                         sx_sunlock(&sc->sc_lock);
1319                         return (error);
1320                 }
1321
1322                 sx_sunlock(&sc->sc_lock);
1323                 return (0);
1324         }
1325 }
1326
1327 static int
1328 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1329     struct iscsi_daemon_handoff *handoff)
1330 {
1331         struct iscsi_session *is;
1332         int error;
1333
1334         sx_slock(&sc->sc_lock);
1335
1336         /*
1337          * Find the session to hand off socket to.
1338          */
1339         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1340                 if (is->is_id == handoff->idh_session_id)
1341                         break;
1342         }
1343         if (is == NULL) {
1344                 sx_sunlock(&sc->sc_lock);
1345                 return (ESRCH);
1346         }
1347         ISCSI_SESSION_LOCK(is);
1348         if (is->is_conf.isc_discovery || is->is_terminating) {
1349                 ISCSI_SESSION_UNLOCK(is);
1350                 sx_sunlock(&sc->sc_lock);
1351                 return (EINVAL);
1352         }
1353         if (is->is_connected) {
1354                 /*
1355                  * This might have happened because another iscsid(8)
1356                  * instance handed off the connection in the meantime.
1357                  * Just return.
1358                  */
1359                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1360                     "session");
1361                 ISCSI_SESSION_UNLOCK(is);
1362                 sx_sunlock(&sc->sc_lock);
1363                 return (EBUSY);
1364         }
1365
1366         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1367             sizeof(is->is_target_alias));
1368         is->is_tsih = handoff->idh_tsih;
1369         is->is_statsn = handoff->idh_statsn;
1370         is->is_initial_r2t = handoff->idh_initial_r2t;
1371         is->is_immediate_data = handoff->idh_immediate_data;
1372         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1373         is->is_max_burst_length = handoff->idh_max_burst_length;
1374         is->is_first_burst_length = handoff->idh_first_burst_length;
1375
1376         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1377                 is->is_conn->ic_header_crc32c = true;
1378         else
1379                 is->is_conn->ic_header_crc32c = false;
1380         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1381                 is->is_conn->ic_data_crc32c = true;
1382         else
1383                 is->is_conn->ic_data_crc32c = false;
1384
1385         is->is_cmdsn = 0;
1386         is->is_expcmdsn = 0;
1387         is->is_maxcmdsn = 0;
1388         is->is_waiting_for_iscsid = false;
1389         is->is_login_phase = false;
1390         is->is_timeout = 0;
1391         is->is_connected = true;
1392         is->is_reason[0] = '\0';
1393
1394         ISCSI_SESSION_UNLOCK(is);
1395
1396 #ifdef ICL_KERNEL_PROXY
1397         if (handoff->idh_socket != 0) {
1398 #endif
1399                 /*
1400                  * Handoff without using ICL proxy.
1401                  */
1402                 error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1403                 if (error != 0) {
1404                         sx_sunlock(&sc->sc_lock);
1405                         iscsi_session_terminate(is);
1406                         return (error);
1407                 }
1408 #ifdef ICL_KERNEL_PROXY
1409         }
1410 #endif
1411
1412         sx_sunlock(&sc->sc_lock);
1413
1414         if (is->is_sim != NULL) {
1415                 /*
1416                  * When reconnecting, there already is SIM allocated for the session.
1417                  */
1418                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1419                 ISCSI_SESSION_LOCK(is);
1420                 ISCSI_SESSION_DEBUG(is, "releasing");
1421                 xpt_release_simq(is->is_sim, 1);
1422                 is->is_simq_frozen = false;
1423                 ISCSI_SESSION_UNLOCK(is);
1424
1425         } else {
1426                 ISCSI_SESSION_LOCK(is);
1427                 is->is_devq = cam_simq_alloc(maxtags);
1428                 if (is->is_devq == NULL) {
1429                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1430                         iscsi_session_terminate(is);
1431                         return (ENOMEM);
1432                 }
1433
1434                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1435                     is, is->is_id /* unit */, &is->is_lock,
1436                     1, maxtags, is->is_devq);
1437                 if (is->is_sim == NULL) {
1438                         ISCSI_SESSION_UNLOCK(is);
1439                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1440                         cam_simq_free(is->is_devq);
1441                         iscsi_session_terminate(is);
1442                         return (ENOMEM);
1443                 }
1444
1445                 error = xpt_bus_register(is->is_sim, NULL, 0);
1446                 if (error != 0) {
1447                         ISCSI_SESSION_UNLOCK(is);
1448                         ISCSI_SESSION_WARN(is, "failed to register bus");
1449                         iscsi_session_terminate(is);
1450                         return (ENOMEM);
1451                 }
1452
1453                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1454                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1455                     CAM_LUN_WILDCARD);
1456                 if (error != CAM_REQ_CMP) {
1457                         ISCSI_SESSION_UNLOCK(is);
1458                         ISCSI_SESSION_WARN(is, "failed to create path");
1459                         iscsi_session_terminate(is);
1460                         return (ENOMEM);
1461                 }
1462                 ISCSI_SESSION_UNLOCK(is);
1463         }
1464
1465         return (0);
1466 }
1467
1468 static int
1469 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1470     struct iscsi_daemon_fail *fail)
1471 {
1472         struct iscsi_session *is;
1473
1474         sx_slock(&sc->sc_lock);
1475
1476         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1477                 if (is->is_id == fail->idf_session_id)
1478                         break;
1479         }
1480         if (is == NULL) {
1481                 sx_sunlock(&sc->sc_lock);
1482                 return (ESRCH);
1483         }
1484         ISCSI_SESSION_LOCK(is);
1485         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1486             fail->idf_reason);
1487         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1488         //is->is_waiting_for_iscsid = false;
1489         //is->is_login_phase = true;
1490         //iscsi_session_reconnect(is);
1491         ISCSI_SESSION_UNLOCK(is);
1492         sx_sunlock(&sc->sc_lock);
1493
1494         return (0);
1495 }
1496
1497 #ifdef ICL_KERNEL_PROXY
1498 static int
1499 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1500     struct iscsi_daemon_connect *idc)
1501 {
1502         struct iscsi_session *is;
1503         struct sockaddr *from_sa, *to_sa;
1504         int error;
1505
1506         sx_slock(&sc->sc_lock);
1507         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1508                 if (is->is_id == idc->idc_session_id)
1509                         break;
1510         }
1511         if (is == NULL) {
1512                 sx_sunlock(&sc->sc_lock);
1513                 return (ESRCH);
1514         }
1515         sx_sunlock(&sc->sc_lock);
1516
1517         if (idc->idc_from_addrlen > 0) {
1518                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1519                 if (error != 0) {
1520                         ISCSI_SESSION_WARN(is,
1521                             "getsockaddr failed with error %d", error);
1522                         return (error);
1523                 }
1524         } else {
1525                 from_sa = NULL;
1526         }
1527         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1528         if (error != 0) {
1529                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1530                     error);
1531                 free(from_sa, M_SONAME);
1532                 return (error);
1533         }
1534
1535         ISCSI_SESSION_LOCK(is);
1536         is->is_waiting_for_iscsid = false;
1537         is->is_login_phase = true;
1538         is->is_timeout = 0;
1539         ISCSI_SESSION_UNLOCK(is);
1540
1541         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1542             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1543         free(from_sa, M_SONAME);
1544         free(to_sa, M_SONAME);
1545
1546         /*
1547          * Digests are always disabled during login phase.
1548          */
1549         is->is_conn->ic_header_crc32c = false;
1550         is->is_conn->ic_data_crc32c = false;
1551
1552         return (error);
1553 }
1554
1555 static int
1556 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1557     struct iscsi_daemon_send *ids)
1558 {
1559         struct iscsi_session *is;
1560         struct icl_pdu *ip;
1561         size_t datalen;
1562         void *data;
1563         int error;
1564
1565         sx_slock(&sc->sc_lock);
1566         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1567                 if (is->is_id == ids->ids_session_id)
1568                         break;
1569         }
1570         if (is == NULL) {
1571                 sx_sunlock(&sc->sc_lock);
1572                 return (ESRCH);
1573         }
1574         sx_sunlock(&sc->sc_lock);
1575
1576         if (is->is_login_phase == false)
1577                 return (EBUSY);
1578
1579         if (is->is_terminating || is->is_reconnecting)
1580                 return (EIO);
1581
1582         datalen = ids->ids_data_segment_len;
1583         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1584                 return (EINVAL);
1585         if (datalen > 0) {
1586                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1587                 error = copyin(ids->ids_data_segment, data, datalen);
1588                 if (error != 0) {
1589                         free(data, M_ISCSI);
1590                         return (error);
1591                 }
1592         }
1593
1594         ip = icl_pdu_new(is->is_conn, M_WAITOK);
1595         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1596         if (datalen > 0) {
1597                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1598                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1599                 free(data, M_ISCSI);
1600         }
1601         icl_pdu_queue(ip);
1602
1603         return (0);
1604 }
1605
1606 static int
1607 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1608     struct iscsi_daemon_receive *idr)
1609 {
1610         struct iscsi_session *is;
1611         struct icl_pdu *ip;
1612         void *data;
1613
1614         sx_slock(&sc->sc_lock);
1615         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1616                 if (is->is_id == idr->idr_session_id)
1617                         break;
1618         }
1619         if (is == NULL) {
1620                 sx_sunlock(&sc->sc_lock);
1621                 return (ESRCH);
1622         }
1623         sx_sunlock(&sc->sc_lock);
1624
1625         if (is->is_login_phase == false)
1626                 return (EBUSY);
1627
1628         ISCSI_SESSION_LOCK(is);
1629         while (is->is_login_pdu == NULL &&
1630             is->is_terminating == false &&
1631             is->is_reconnecting == false)
1632                 cv_wait(&is->is_login_cv, &is->is_lock);
1633         if (is->is_terminating || is->is_reconnecting) {
1634                 ISCSI_SESSION_UNLOCK(is);
1635                 return (EIO);
1636         }
1637         ip = is->is_login_pdu;
1638         is->is_login_pdu = NULL;
1639         ISCSI_SESSION_UNLOCK(is);
1640
1641         if (ip->ip_data_len > idr->idr_data_segment_len) {
1642                 icl_pdu_free(ip);
1643                 return (EMSGSIZE);
1644         }
1645
1646         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1647         if (ip->ip_data_len > 0) {
1648                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1649                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1650                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1651                 free(data, M_ISCSI);
1652         }
1653
1654         icl_pdu_free(ip);
1655
1656         return (0);
1657 }
1658 #endif /* ICL_KERNEL_PROXY */
1659
1660 static void
1661 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1662 {
1663         /*
1664          * Just make sure all the fields are null-terminated.
1665          *
1666          * XXX: This is not particularly secure.  We should
1667          *      create our own conf and then copy in relevant
1668          *      fields.
1669          */
1670         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1671         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1672         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1673         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1674         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1675         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1676         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1677         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1678         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1679 }
1680
1681 static bool
1682 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1683 {
1684
1685         if (isc->isc_initiator[0] == '\0') {
1686                 ISCSI_DEBUG("empty isc_initiator");
1687                 return (false);
1688         }
1689
1690         if (isc->isc_target_addr[0] == '\0') {
1691                 ISCSI_DEBUG("empty isc_target_addr");
1692                 return (false);
1693         }
1694
1695         if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1696                 ISCSI_DEBUG("non-empty isc_target for discovery session");
1697                 return (false);
1698         }
1699
1700         if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1701                 ISCSI_DEBUG("empty isc_target for non-discovery session");
1702                 return (false);
1703         }
1704
1705         return (true);
1706 }
1707
1708 static int
1709 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1710 {
1711         struct iscsi_session *is;
1712         const struct iscsi_session *is2;
1713         int error;
1714
1715         iscsi_sanitize_session_conf(&isa->isa_conf);
1716         if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1717                 return (EINVAL);
1718
1719         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1720         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1721
1722         sx_xlock(&sc->sc_lock);
1723
1724         /*
1725          * Prevent duplicates.
1726          */
1727         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1728                 if (!!is->is_conf.isc_discovery !=
1729                     !!is2->is_conf.isc_discovery)
1730                         continue;
1731
1732                 if (strcmp(is->is_conf.isc_target_addr,
1733                     is2->is_conf.isc_target_addr) != 0)
1734                         continue;
1735
1736                 if (is->is_conf.isc_discovery == 0 &&
1737                     strcmp(is->is_conf.isc_target,
1738                     is2->is_conf.isc_target) != 0)
1739                         continue;
1740
1741                 sx_xunlock(&sc->sc_lock);
1742                 free(is, M_ISCSI);
1743                 return (EBUSY);
1744         }
1745
1746         is->is_conn = icl_new_conn(is->is_conf.isc_offload,
1747             "iscsi", &is->is_lock);
1748         if (is->is_conn == NULL) {
1749                 sx_xunlock(&sc->sc_lock);
1750                 free(is, M_ISCSI);
1751                 return (EINVAL);
1752         }
1753         is->is_conn->ic_receive = iscsi_receive_callback;
1754         is->is_conn->ic_error = iscsi_error_callback;
1755         is->is_conn->ic_prv0 = is;
1756         TAILQ_INIT(&is->is_outstanding);
1757         STAILQ_INIT(&is->is_postponed);
1758         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1759         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1760 #ifdef ICL_KERNEL_PROXY
1761         cv_init(&is->is_login_cv, "iscsi_login");
1762 #endif
1763
1764         is->is_softc = sc;
1765         sc->sc_last_session_id++;
1766         is->is_id = sc->sc_last_session_id;
1767         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1768         arc4rand(&is->is_isid[1], 5, 0);
1769         is->is_tsih = 0;
1770         callout_init(&is->is_callout, 1);
1771
1772         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1773         if (error != 0) {
1774                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1775                 sx_xunlock(&sc->sc_lock);
1776                 return (error);
1777         }
1778
1779         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1780         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1781
1782         /*
1783          * Trigger immediate reconnection.
1784          */
1785         ISCSI_SESSION_LOCK(is);
1786         is->is_waiting_for_iscsid = true;
1787         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1788         ISCSI_SESSION_UNLOCK(is);
1789         cv_signal(&sc->sc_cv);
1790
1791         sx_xunlock(&sc->sc_lock);
1792
1793         return (0);
1794 }
1795
1796 static bool
1797 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1798     unsigned int id2, const struct iscsi_session_conf *c2)
1799 {
1800
1801         if (id2 != 0 && id2 != id1)
1802                 return (false);
1803         if (c2->isc_target[0] != '\0' &&
1804             strcmp(c1->isc_target, c2->isc_target) != 0)
1805                 return (false);
1806         if (c2->isc_target_addr[0] != '\0' &&
1807             strcmp(c1->isc_target_addr, c2->isc_target_addr) != 0)
1808                 return (false);
1809         return (true);
1810 }
1811
1812 static int
1813 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1814     struct iscsi_session_remove *isr)
1815 {
1816         struct iscsi_session *is, *tmp;
1817         bool found = false;
1818
1819         iscsi_sanitize_session_conf(&isr->isr_conf);
1820
1821         sx_xlock(&sc->sc_lock);
1822         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1823                 ISCSI_SESSION_LOCK(is);
1824                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1825                     isr->isr_session_id, &isr->isr_conf)) {
1826                         found = true;
1827                         iscsi_session_logout(is);
1828                         iscsi_session_terminate(is);
1829                 }
1830                 ISCSI_SESSION_UNLOCK(is);
1831         }
1832         sx_xunlock(&sc->sc_lock);
1833
1834         if (!found)
1835                 return (ESRCH);
1836
1837         return (0);
1838 }
1839
1840 static int
1841 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1842 {
1843         int error;
1844         unsigned int i = 0;
1845         struct iscsi_session *is;
1846         struct iscsi_session_state iss;
1847
1848         sx_slock(&sc->sc_lock);
1849         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1850                 if (i >= isl->isl_nentries) {
1851                         sx_sunlock(&sc->sc_lock);
1852                         return (EMSGSIZE);
1853                 }
1854                 memset(&iss, 0, sizeof(iss));
1855                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1856                 iss.iss_id = is->is_id;
1857                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1858                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1859                 strlcpy(iss.iss_offload, is->is_conn->ic_offload, sizeof(iss.iss_offload));
1860
1861                 if (is->is_conn->ic_header_crc32c)
1862                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1863                 else
1864                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1865
1866                 if (is->is_conn->ic_data_crc32c)
1867                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1868                 else
1869                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1870
1871                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1872                 iss.iss_immediate_data = is->is_immediate_data;
1873                 iss.iss_connected = is->is_connected;
1874
1875                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1876                 if (error != 0) {
1877                         sx_sunlock(&sc->sc_lock);
1878                         return (error);
1879                 }
1880                 i++;
1881         }
1882         sx_sunlock(&sc->sc_lock);
1883
1884         isl->isl_nentries = i;
1885
1886         return (0);
1887 }
1888
1889 static int
1890 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
1891     struct iscsi_session_modify *ism)
1892 {
1893         struct iscsi_session *is;
1894
1895         iscsi_sanitize_session_conf(&ism->ism_conf);
1896         if (iscsi_valid_session_conf(&ism->ism_conf) == false)
1897                 return (EINVAL);
1898
1899         sx_xlock(&sc->sc_lock);
1900         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1901                 ISCSI_SESSION_LOCK(is);
1902                 if (is->is_id == ism->ism_session_id)
1903                         break;
1904                 ISCSI_SESSION_UNLOCK(is);
1905         }
1906         if (is == NULL) {
1907                 sx_xunlock(&sc->sc_lock);
1908                 return (ESRCH);
1909         }
1910         sx_xunlock(&sc->sc_lock);
1911
1912         memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
1913         ISCSI_SESSION_UNLOCK(is);
1914
1915         iscsi_session_reconnect(is);
1916
1917         return (0);
1918 }
1919
1920 static int
1921 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1922     struct thread *td)
1923 {
1924         struct iscsi_softc *sc;
1925
1926         sc = dev->si_drv1;
1927
1928         switch (cmd) {
1929         case ISCSIDWAIT:
1930                 return (iscsi_ioctl_daemon_wait(sc,
1931                     (struct iscsi_daemon_request *)arg));
1932         case ISCSIDHANDOFF:
1933                 return (iscsi_ioctl_daemon_handoff(sc,
1934                     (struct iscsi_daemon_handoff *)arg));
1935         case ISCSIDFAIL:
1936                 return (iscsi_ioctl_daemon_fail(sc,
1937                     (struct iscsi_daemon_fail *)arg));
1938 #ifdef ICL_KERNEL_PROXY
1939         case ISCSIDCONNECT:
1940                 return (iscsi_ioctl_daemon_connect(sc,
1941                     (struct iscsi_daemon_connect *)arg));
1942         case ISCSIDSEND:
1943                 return (iscsi_ioctl_daemon_send(sc,
1944                     (struct iscsi_daemon_send *)arg));
1945         case ISCSIDRECEIVE:
1946                 return (iscsi_ioctl_daemon_receive(sc,
1947                     (struct iscsi_daemon_receive *)arg));
1948 #endif /* ICL_KERNEL_PROXY */
1949         case ISCSISADD:
1950                 return (iscsi_ioctl_session_add(sc,
1951                     (struct iscsi_session_add *)arg));
1952         case ISCSISREMOVE:
1953                 return (iscsi_ioctl_session_remove(sc,
1954                     (struct iscsi_session_remove *)arg));
1955         case ISCSISLIST:
1956                 return (iscsi_ioctl_session_list(sc,
1957                     (struct iscsi_session_list *)arg));
1958         case ISCSISMODIFY:
1959                 return (iscsi_ioctl_session_modify(sc,
1960                     (struct iscsi_session_modify *)arg));
1961         default:
1962                 return (EINVAL);
1963         }
1964 }
1965
1966 static struct iscsi_outstanding *
1967 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1968 {
1969         struct iscsi_outstanding *io;
1970
1971         ISCSI_SESSION_LOCK_ASSERT(is);
1972
1973         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1974                 if (io->io_initiator_task_tag == initiator_task_tag)
1975                         return (io);
1976         }
1977         return (NULL);
1978 }
1979
1980 static struct iscsi_outstanding *
1981 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
1982 {
1983         struct iscsi_outstanding *io;
1984
1985         ISCSI_SESSION_LOCK_ASSERT(is);
1986
1987         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1988                 if (io->io_ccb == ccb)
1989                         return (io);
1990         }
1991         return (NULL);
1992 }
1993
1994 static struct iscsi_outstanding *
1995 iscsi_outstanding_add(struct iscsi_session *is,
1996     union ccb *ccb, uint32_t *initiator_task_tagp)
1997 {
1998         struct iscsi_outstanding *io;
1999         int error;
2000
2001         ISCSI_SESSION_LOCK_ASSERT(is);
2002
2003         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
2004         if (io == NULL) {
2005                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes",
2006                     sizeof(*io));
2007                 return (NULL);
2008         }
2009
2010         error = icl_conn_task_setup(is->is_conn, &ccb->csio,
2011             initiator_task_tagp, &io->io_icl_prv);
2012         if (error != 0) {
2013                 ISCSI_SESSION_WARN(is,
2014                     "icl_conn_task_setup() failed with error %d", error);
2015                 uma_zfree(iscsi_outstanding_zone, io);
2016                 return (NULL);
2017         }
2018
2019         KASSERT(iscsi_outstanding_find(is, *initiator_task_tagp) == NULL,
2020             ("initiator_task_tag 0x%x already added", *initiator_task_tagp));
2021
2022         io->io_initiator_task_tag = *initiator_task_tagp;
2023         io->io_ccb = ccb;
2024         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
2025         return (io);
2026 }
2027
2028 static void
2029 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
2030 {
2031
2032         ISCSI_SESSION_LOCK_ASSERT(is);
2033
2034         icl_conn_task_done(is->is_conn, io->io_icl_prv);
2035         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
2036         uma_zfree(iscsi_outstanding_zone, io);
2037 }
2038
2039 static void
2040 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
2041 {
2042         struct icl_pdu *request;
2043         struct iscsi_bhs_task_management_request *bhstmr;
2044         struct ccb_abort *cab = &ccb->cab;
2045         struct iscsi_outstanding *io, *aio;
2046         uint32_t initiator_task_tag;
2047
2048         ISCSI_SESSION_LOCK_ASSERT(is);
2049
2050 #if 0
2051         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2052 #else
2053         if (is->is_login_phase) {
2054                 ccb->ccb_h.status = CAM_REQ_ABORTED;
2055                 xpt_done(ccb);
2056                 return;
2057         }
2058 #endif
2059
2060         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
2061         if (aio == NULL) {
2062                 ccb->ccb_h.status = CAM_REQ_CMP;
2063                 xpt_done(ccb);
2064                 return;
2065         }
2066
2067         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2068         if (request == NULL) {
2069                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2070                 xpt_done(ccb);
2071                 return;
2072         }
2073
2074         initiator_task_tag = is->is_initiator_task_tag++;
2075
2076         io = iscsi_outstanding_add(is, NULL, &initiator_task_tag);
2077         if (io == NULL) {
2078                 icl_pdu_free(request);
2079                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2080                 xpt_done(ccb);
2081                 return;
2082         }
2083         io->io_datasn = aio->io_initiator_task_tag;
2084
2085         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2086         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
2087         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
2088         bhstmr->bhstmr_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
2089         bhstmr->bhstmr_initiator_task_tag = initiator_task_tag;
2090         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2091
2092         iscsi_pdu_queue_locked(request);
2093 }
2094
2095 static void
2096 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2097 {
2098         struct icl_pdu *request;
2099         struct iscsi_bhs_scsi_command *bhssc;
2100         struct ccb_scsiio *csio;
2101         struct iscsi_outstanding *io;
2102         size_t len;
2103         uint32_t initiator_task_tag;
2104         int error;
2105
2106         ISCSI_SESSION_LOCK_ASSERT(is);
2107
2108 #if 0
2109         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2110 #else
2111         if (is->is_login_phase) {
2112                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2113                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2114                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2115                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2116                 }
2117                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2118                 xpt_done(ccb);
2119                 return;
2120         }
2121 #endif
2122
2123         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2124         if (request == NULL) {
2125                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2126                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2127                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2128                 }
2129                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2130                 xpt_done(ccb);
2131                 return;
2132         }
2133
2134         initiator_task_tag = is->is_initiator_task_tag++;
2135         io = iscsi_outstanding_add(is, ccb, &initiator_task_tag);
2136         if (io == NULL) {
2137                 icl_pdu_free(request);
2138                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2139                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2140                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2141                 }
2142                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2143                 xpt_done(ccb);
2144                 return;
2145         }
2146
2147         csio = &ccb->csio;
2148         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2149         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2150         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2151         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2152         case CAM_DIR_IN:
2153                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2154                 break;
2155         case CAM_DIR_OUT:
2156                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2157                 break;
2158         }
2159
2160         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2161                 switch (csio->tag_action) {
2162                 case MSG_HEAD_OF_Q_TAG:
2163                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2164                         break;
2165                 case MSG_ORDERED_Q_TAG:
2166                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2167                         break;
2168                 case MSG_ACA_TASK:
2169                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2170                         break;
2171                 case MSG_SIMPLE_Q_TAG:
2172                 default:
2173                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2174                         break;
2175                 }
2176         } else
2177                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2178
2179         bhssc->bhssc_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
2180         bhssc->bhssc_initiator_task_tag = initiator_task_tag;
2181         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2182         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2183             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2184
2185         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2186                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2187         else
2188                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2189
2190         if (is->is_immediate_data &&
2191             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2192                 len = csio->dxfer_len;
2193                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2194                 if (len > is->is_first_burst_length) {
2195                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
2196                         len = is->is_first_burst_length;
2197                 }
2198                 if (len > is->is_max_data_segment_length) {
2199                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_max_data_segment_length);
2200                         len = is->is_max_data_segment_length;
2201                 }
2202
2203                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2204                 if (error != 0) {
2205                         icl_pdu_free(request);
2206                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2207                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2208                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2209                         }
2210                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2211                         xpt_done(ccb);
2212                         return;
2213                 }
2214         }
2215         iscsi_pdu_queue_locked(request);
2216 }
2217
2218 static void
2219 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2220 {
2221         struct iscsi_session *is;
2222
2223         is = cam_sim_softc(sim);
2224
2225         ISCSI_SESSION_LOCK_ASSERT(is);
2226
2227         if (is->is_terminating ||
2228             (is->is_connected == false && fail_on_disconnection)) {
2229                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2230                 xpt_done(ccb);
2231                 return;
2232         }
2233
2234         switch (ccb->ccb_h.func_code) {
2235         case XPT_PATH_INQ:
2236         {
2237                 struct ccb_pathinq *cpi = &ccb->cpi;
2238
2239                 cpi->version_num = 1;
2240                 cpi->hba_inquiry = PI_TAG_ABLE;
2241                 cpi->target_sprt = 0;
2242                 cpi->hba_misc = PIM_EXTLUNS;
2243                 cpi->hba_eng_cnt = 0;
2244                 cpi->max_target = 0;
2245                 /*
2246                  * Note that the variable below is only relevant for targets
2247                  * that don't claim compliance with anything above SPC2, which
2248                  * means they don't support REPORT_LUNS.
2249                  */
2250                 cpi->max_lun = 255;
2251                 cpi->initiator_id = ~0;
2252                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2253                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2254                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2255                 cpi->unit_number = cam_sim_unit(sim);
2256                 cpi->bus_id = cam_sim_bus(sim);
2257                 cpi->base_transfer_speed = 150000; /* XXX */
2258                 cpi->transport = XPORT_ISCSI;
2259                 cpi->transport_version = 0;
2260                 cpi->protocol = PROTO_SCSI;
2261                 cpi->protocol_version = SCSI_REV_SPC3;
2262                 cpi->maxio = MAXPHYS;
2263                 cpi->ccb_h.status = CAM_REQ_CMP;
2264                 break;
2265         }
2266         case XPT_GET_TRAN_SETTINGS:
2267         {
2268                 struct ccb_trans_settings       *cts;
2269                 struct ccb_trans_settings_scsi  *scsi;
2270
2271                 cts = &ccb->cts;
2272                 scsi = &cts->proto_specific.scsi;
2273
2274                 cts->protocol = PROTO_SCSI;
2275                 cts->protocol_version = SCSI_REV_SPC3;
2276                 cts->transport = XPORT_ISCSI;
2277                 cts->transport_version = 0;
2278                 scsi->valid = CTS_SCSI_VALID_TQ;
2279                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2280                 cts->ccb_h.status = CAM_REQ_CMP;
2281                 break;
2282         }
2283         case XPT_CALC_GEOMETRY:
2284                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2285                 ccb->ccb_h.status = CAM_REQ_CMP;
2286                 break;
2287 #if 0
2288         /*
2289          * XXX: What's the point?
2290          */
2291         case XPT_RESET_BUS:
2292         case XPT_TERM_IO:
2293                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2294                 ccb->ccb_h.status = CAM_REQ_CMP;
2295                 break;
2296 #endif
2297         case XPT_ABORT:
2298                 iscsi_action_abort(is, ccb);
2299                 return;
2300         case XPT_SCSI_IO:
2301                 iscsi_action_scsiio(is, ccb);
2302                 return;
2303         default:
2304 #if 0
2305                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2306 #endif
2307                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2308                 break;
2309         }
2310         xpt_done(ccb);
2311 }
2312
2313 static void
2314 iscsi_poll(struct cam_sim *sim)
2315 {
2316
2317         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2318 }
2319
2320 static void
2321 iscsi_shutdown(struct iscsi_softc *sc)
2322 {
2323         struct iscsi_session *is;
2324
2325         /*
2326          * Trying to reconnect during system shutdown would lead to hang.
2327          */
2328         fail_on_disconnection = 1;
2329
2330         /*
2331          * If we have any sessions waiting for reconnection, request
2332          * maintenance thread to fail them immediately instead of waiting
2333          * for reconnect timeout.
2334          */
2335         sx_slock(&sc->sc_lock);
2336         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
2337                 ISCSI_SESSION_LOCK(is);
2338                 if (is->is_waiting_for_iscsid)
2339                         iscsi_session_reconnect(is);
2340                 ISCSI_SESSION_UNLOCK(is);
2341         }
2342         sx_sunlock(&sc->sc_lock);
2343 }
2344
2345 static int
2346 iscsi_load(void)
2347 {
2348         int error;
2349
2350         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2351         sx_init(&sc->sc_lock, "iscsi");
2352         TAILQ_INIT(&sc->sc_sessions);
2353         cv_init(&sc->sc_cv, "iscsi_cv");
2354
2355         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2356             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2357             UMA_ALIGN_PTR, 0);
2358
2359         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2360             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2361         if (error != 0) {
2362                 ISCSI_WARN("failed to create device node, error %d", error);
2363                 return (error);
2364         }
2365         sc->sc_cdev->si_drv1 = sc;
2366
2367         sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
2368             iscsi_shutdown, sc, SHUTDOWN_PRI_DEFAULT-1);
2369
2370         return (0);
2371 }
2372
2373 static int
2374 iscsi_unload(void)
2375 {
2376         struct iscsi_session *is, *tmp;
2377
2378         if (sc->sc_cdev != NULL) {
2379                 ISCSI_DEBUG("removing device node");
2380                 destroy_dev(sc->sc_cdev);
2381                 ISCSI_DEBUG("device node removed");
2382         }
2383
2384         if (sc->sc_shutdown_eh != NULL)
2385                 EVENTHANDLER_DEREGISTER(shutdown_pre_sync, sc->sc_shutdown_eh);
2386
2387         sx_slock(&sc->sc_lock);
2388         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2389                 iscsi_session_terminate(is);
2390         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2391                 ISCSI_DEBUG("waiting for sessions to terminate");
2392                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2393         }
2394         ISCSI_DEBUG("all sessions terminated");
2395         sx_sunlock(&sc->sc_lock);
2396
2397         uma_zdestroy(iscsi_outstanding_zone);
2398         sx_destroy(&sc->sc_lock);
2399         cv_destroy(&sc->sc_cv);
2400         free(sc, M_ISCSI);
2401         return (0);
2402 }
2403
2404 static int
2405 iscsi_quiesce(void)
2406 {
2407         sx_slock(&sc->sc_lock);
2408         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2409                 sx_sunlock(&sc->sc_lock);
2410                 return (EBUSY);
2411         }
2412         sx_sunlock(&sc->sc_lock);
2413         return (0);
2414 }
2415
2416 static int
2417 iscsi_modevent(module_t mod, int what, void *arg)
2418 {
2419         int error;
2420
2421         switch (what) {
2422         case MOD_LOAD:
2423                 error = iscsi_load();
2424                 break;
2425         case MOD_UNLOAD:
2426                 error = iscsi_unload();
2427                 break;
2428         case MOD_QUIESCE:
2429                 error = iscsi_quiesce();
2430                 break;
2431         default:
2432                 error = EINVAL;
2433                 break;
2434         }
2435         return (error);
2436 }
2437
2438 moduledata_t iscsi_data = {
2439         "iscsi",
2440         iscsi_modevent,
2441         0
2442 };
2443
2444 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2445 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2446 MODULE_DEPEND(iscsi, icl, 1, 1, 1);