sys/dev/iscsi/iscsi.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012 The FreeBSD Foundation
   5  *
   6  * This software was developed by Edward Tomasz Napierala under sponsorship
   7  * from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include <sys/param.h>
  36 #include <sys/bio.h>
  37 #include <sys/condvar.h>
  38 #include <sys/conf.h>
  39 #include <sys/endian.h>
  40 #include <sys/eventhandler.h>
  41 #include <sys/file.h>
  42 #include <sys/kernel.h>
  43 #include <sys/kthread.h>
  44 #include <sys/lock.h>
  45 #include <sys/malloc.h>
  46 #include <sys/mbuf.h>
  47 #include <sys/mutex.h>
  48 #include <sys/module.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockopt.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/systm.h>
  53 #include <sys/sx.h>
  54 #include <vm/uma.h>
  55
  56 #include <cam/cam.h>
  57 #include <cam/cam_ccb.h>
  58 #include <cam/cam_xpt.h>
  59 #include <cam/cam_debug.h>
  60 #include <cam/cam_sim.h>
  61 #include <cam/cam_xpt_sim.h>
  62 #include <cam/cam_xpt_periph.h>
  63 #include <cam/cam_periph.h>
  64 #include <cam/scsi/scsi_all.h>
  65 #include <cam/scsi/scsi_message.h>
  66
  67 #include <dev/iscsi/icl.h>
  68 #include <dev/iscsi/icl_wrappers.h>
  69 #include <dev/iscsi/iscsi_ioctl.h>
  70 #include <dev/iscsi/iscsi_proto.h>
  71 #include <dev/iscsi/iscsi.h>
  72
  73 #ifdef ICL_KERNEL_PROXY
  74 #include <sys/socketvar.h>
  75 #endif
  76
  77 #ifdef ICL_KERNEL_PROXY
  78 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  79 #endif
  80
  81 /*
  82  * XXX: This is global so the iscsi_unload() can access it.
  83  *      Think about how to do this properly.
  84  */
  85 static struct iscsi_softc       *sc;
  86
  87 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  88     "iSCSI initiator");
  89 static int debug = 1;
  90 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  91     &debug, 0, "Enable debug messages");
  92
  93 static int ping_timeout = 5;
  94 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  95     0, "Timeout for ping (NOP-Out) requests, in seconds");
  96 static int iscsid_timeout = 60;
  97 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  98     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  99 static int login_timeout = 60;
 100 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
 101     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
 102 static int maxtags = 255;
 103 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
 104     0, "Max number of IO requests queued");
 105 static int fail_on_disconnection = 0;
 106 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
 107     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
 108 static int fail_on_shutdown = 1;
 109 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_shutdown, CTLFLAG_RWTUN,
 110     &fail_on_shutdown, 0, "Fail disconnected sessions on shutdown");
 111
 112 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 113 static uma_zone_t iscsi_outstanding_zone;
 114
 115 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 116 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 117
 118 #define ISCSI_DEBUG(X, ...)                                             \
 119         do {                                                            \
 120                 if (debug > 1)                                          \
 121                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 122         } while (0)
 123
 124 #define ISCSI_WARN(X, ...)                                              \
 125         do {                                                            \
 126                 if (debug > 0) {                                        \
 127                         printf("WARNING: %s: " X "\n",                  \
 128                             __func__, ## __VA_ARGS__);                  \
 129                 }                                                       \
 130         } while (0)
 131
 132 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 133         do {                                                            \
 134                 if (debug > 1) {                                        \
 135                         printf("%s: %s (%s): " X "\n",                  \
 136                             __func__, S->is_conf.isc_target_addr,       \
 137                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 138                 }                                                       \
 139         } while (0)
 140
 141 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 142         do {                                                            \
 143                 if (debug > 0) {                                        \
 144                         printf("WARNING: %s (%s): " X "\n",             \
 145                             S->is_conf.isc_target_addr,                 \
 146                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 147                 }                                                       \
 148         } while (0)
 149
 150 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 151 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 152 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 153 #define ISCSI_SESSION_LOCK_ASSERT_NOT(X) mtx_assert(&X->is_lock, MA_NOTOWNED)
 154
 155 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 156                     int mode, struct thread *td);
 157
 158 static struct cdevsw iscsi_cdevsw = {
 159      .d_version = D_VERSION,
 160      .d_ioctl   = iscsi_ioctl,
 161      .d_name    = "iscsi",
 162 };
 163
 164 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 165 static void     iscsi_pdu_queue(struct icl_pdu *request);
 166 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 167 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 168 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 169 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
 170 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 171 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 172 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 173 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 174 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 175 static void     iscsi_session_reconnect(struct iscsi_session *is);
 176 static void     iscsi_session_terminate(struct iscsi_session *is);
 177 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 178 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 179                     uint32_t initiator_task_tag);
 180 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
 181                     struct icl_pdu *request, union ccb *ccb,
 182                     uint32_t *initiator_task_tagp);
 183 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 184                     struct iscsi_outstanding *io);
 185
 186 static bool
 187 iscsi_pdu_prepare(struct icl_pdu *request)
 188 {
 189         struct iscsi_session *is;
 190         struct iscsi_bhs_scsi_command *bhssc;
 191
 192         is = PDU_SESSION(request);
 193
 194         ISCSI_SESSION_LOCK_ASSERT(is);
 195
 196         /*
 197          * We're only using fields common for all the request
 198          * (initiator -> target) PDUs.
 199          */
 200         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 201
 202         /*
 203          * Data-Out PDU does not contain CmdSN.
 204          */
 205         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 206                 if (ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn) &&
 207                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 208                         /*
 209                          * Current MaxCmdSN prevents us from sending any more
 210                          * SCSI Command PDUs to the target; postpone the PDU.
 211                          * It will get resent by either iscsi_pdu_queue(),
 212                          * or by maintenance thread.
 213                          */
 214 #if 0
 215                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %u, "
 216                             "ExpCmdSN %u, MaxCmdSN %u, opcode 0x%x",
 217                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn,
 218                             bhssc->bhssc_opcode);
 219 #endif
 220                         return (true);
 221                 }
 222                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 223                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 224                         is->is_cmdsn++;
 225         }
 226         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 227
 228         return (false);
 229 }
 230
 231 static void
 232 iscsi_session_send_postponed(struct iscsi_session *is)
 233 {
 234         struct icl_pdu *request;
 235         bool postpone;
 236
 237         ISCSI_SESSION_LOCK_ASSERT(is);
 238
 239         if (STAILQ_EMPTY(&is->is_postponed))
 240                 return;
 241         while ((request = STAILQ_FIRST(&is->is_postponed)) != NULL) {
 242                 postpone = iscsi_pdu_prepare(request);
 243                 if (postpone)
 244                         return;
 245                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 246                 icl_pdu_queue(request);
 247         }
 248         xpt_release_simq(is->is_sim, 1);
 249 }
 250
 251 static void
 252 iscsi_pdu_queue_locked(struct icl_pdu *request)
 253 {
 254         struct iscsi_session *is;
 255         bool postpone;
 256
 257         is = PDU_SESSION(request);
 258         ISCSI_SESSION_LOCK_ASSERT(is);
 259         iscsi_session_send_postponed(is);
 260         postpone = iscsi_pdu_prepare(request);
 261         if (postpone) {
 262                 if (STAILQ_EMPTY(&is->is_postponed))
 263                         xpt_freeze_simq(is->is_sim, 1);
 264                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 265                 return;
 266         }
 267         icl_pdu_queue(request);
 268 }
 269
 270 static void
 271 iscsi_pdu_queue(struct icl_pdu *request)
 272 {
 273         struct iscsi_session *is;
 274
 275         is = PDU_SESSION(request);
 276         ISCSI_SESSION_LOCK(is);
 277         iscsi_pdu_queue_locked(request);
 278         ISCSI_SESSION_UNLOCK(is);
 279 }
 280
 281 static void
 282 iscsi_session_logout(struct iscsi_session *is)
 283 {
 284         struct icl_pdu *request;
 285         struct iscsi_bhs_logout_request *bhslr;
 286
 287         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 288         if (request == NULL)
 289                 return;
 290
 291         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 292         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 293         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 294         iscsi_pdu_queue_locked(request);
 295 }
 296
 297 static void
 298 iscsi_session_terminate_task(struct iscsi_session *is,
 299     struct iscsi_outstanding *io, cam_status status)
 300 {
 301
 302         ISCSI_SESSION_LOCK_ASSERT(is);
 303
 304         if (io->io_ccb != NULL) {
 305                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
 306                 io->io_ccb->ccb_h.status |= status;
 307                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 308                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 309                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 310                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 311                 }
 312                 xpt_done(io->io_ccb);
 313         }
 314         iscsi_outstanding_remove(is, io);
 315 }
 316
 317 static void
 318 iscsi_session_terminate_tasks(struct iscsi_session *is, cam_status status)
 319 {
 320         struct iscsi_outstanding *io, *tmp;
 321
 322         ISCSI_SESSION_LOCK_ASSERT(is);
 323
 324         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 325                 iscsi_session_terminate_task(is, io, status);
 326         }
 327 }
 328
 329 static void
 330 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 331 {
 332         struct icl_pdu *pdu;
 333
 334         ISCSI_SESSION_LOCK_ASSERT(is);
 335
 336         /*
 337          * Don't queue any new PDUs.
 338          */
 339         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 340                 ISCSI_SESSION_DEBUG(is, "freezing");
 341                 xpt_freeze_simq(is->is_sim, 1);
 342                 is->is_simq_frozen = true;
 343         }
 344
 345         /*
 346          * Remove postponed PDUs.
 347          */
 348         if (!STAILQ_EMPTY(&is->is_postponed))
 349                 xpt_release_simq(is->is_sim, 1);
 350         while ((pdu = STAILQ_FIRST(&is->is_postponed)) != NULL) {
 351                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 352                 icl_pdu_free(pdu);
 353         }
 354
 355         if (destroy_sim == false) {
 356                 /*
 357                  * Terminate SCSI tasks, asking CAM to requeue them.
 358                  */
 359                 iscsi_session_terminate_tasks(is, CAM_REQUEUE_REQ);
 360                 return;
 361         }
 362
 363         iscsi_session_terminate_tasks(is, CAM_DEV_NOT_THERE);
 364
 365         if (is->is_sim == NULL)
 366                 return;
 367
 368         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 369         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 370
 371         if (is->is_simq_frozen) {
 372                 is->is_simq_frozen = false;
 373                 xpt_release_simq(is->is_sim, 1);
 374         }
 375
 376         xpt_free_path(is->is_path);
 377         is->is_path = NULL;
 378         xpt_bus_deregister(cam_sim_path(is->is_sim));
 379         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 380         is->is_sim = NULL;
 381         is->is_devq = NULL;
 382 }
 383
 384 static void
 385 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 386 {
 387         /*
 388          * As we will be reconnecting shortly,
 389          * discard outstanding data immediately on
 390          * close(), also notify peer via RST if
 391          * any packets come in.
 392          */
 393         struct socket *so;
 394         so = is->is_conn->ic_socket;
 395         if (so != NULL) {
 396                 struct sockopt sopt;
 397                 struct linger sl;
 398                 sopt.sopt_dir     = SOPT_SET;
 399                 sopt.sopt_level   = SOL_SOCKET;
 400                 sopt.sopt_name    = SO_LINGER;
 401                 sopt.sopt_val     = &sl;
 402                 sopt.sopt_valsize = sizeof(sl);
 403                 sl.l_onoff        = 1;  /* non-zero value enables linger option in kernel */
 404                 sl.l_linger       = 0;  /* timeout interval in seconds */
 405                 sosetopt(is->is_conn->ic_socket, &sopt);
 406         }
 407
 408         icl_conn_close(is->is_conn);
 409
 410         ISCSI_SESSION_LOCK(is);
 411
 412         is->is_connected = false;
 413         is->is_reconnecting = false;
 414         is->is_login_phase = false;
 415
 416 #ifdef ICL_KERNEL_PROXY
 417         if (is->is_login_pdu != NULL) {
 418                 icl_pdu_free(is->is_login_pdu);
 419                 is->is_login_pdu = NULL;
 420         }
 421         cv_signal(&is->is_login_cv);
 422 #endif
 423
 424         if (fail_on_disconnection) {
 425                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
 426                 iscsi_session_cleanup(is, true);
 427         } else {
 428                 iscsi_session_cleanup(is, false);
 429         }
 430
 431         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 432             ("destroying session with active tasks"));
 433         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 434             ("destroying session with postponed PDUs"));
 435
 436         if (is->is_conf.isc_enable == 0 && is->is_conf.isc_discovery == 0) {
 437                 ISCSI_SESSION_UNLOCK(is);
 438                 return;
 439         }
 440
 441         /*
 442          * Request immediate reconnection from iscsid(8).
 443          */
 444         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 445         is->is_waiting_for_iscsid = true;
 446         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 447         is->is_timeout = 0;
 448         ISCSI_SESSION_UNLOCK(is);
 449         cv_signal(&is->is_softc->sc_cv);
 450 }
 451
 452 static void
 453 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 454 {
 455         struct iscsi_softc *sc;
 456
 457         sc = is->is_softc;
 458         sx_xlock(&sc->sc_lock);
 459         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 460         sx_xunlock(&sc->sc_lock);
 461
 462         icl_conn_close(is->is_conn);
 463         callout_drain(&is->is_callout);
 464
 465         ISCSI_SESSION_LOCK(is);
 466
 467         KASSERT(is->is_terminating, ("is_terminating == false"));
 468
 469 #ifdef ICL_KERNEL_PROXY
 470         if (is->is_login_pdu != NULL) {
 471                 icl_pdu_free(is->is_login_pdu);
 472                 is->is_login_pdu = NULL;
 473         }
 474         cv_signal(&is->is_login_cv);
 475 #endif
 476
 477         iscsi_session_cleanup(is, true);
 478
 479         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 480             ("destroying session with active tasks"));
 481         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 482             ("destroying session with postponed PDUs"));
 483
 484         ISCSI_SESSION_UNLOCK(is);
 485
 486         icl_conn_free(is->is_conn);
 487         mtx_destroy(&is->is_lock);
 488         cv_destroy(&is->is_maintenance_cv);
 489 #ifdef ICL_KERNEL_PROXY
 490         cv_destroy(&is->is_login_cv);
 491 #endif
 492
 493         ISCSI_SESSION_DEBUG(is, "terminated");
 494         free(is, M_ISCSI);
 495
 496         /*
 497          * The iscsi_unload() routine might be waiting.
 498          */
 499         cv_signal(&sc->sc_cv);
 500 }
 501
 502 static void
 503 iscsi_maintenance_thread(void *arg)
 504 {
 505         struct iscsi_session *is = arg;
 506
 507         ISCSI_SESSION_LOCK(is);
 508         for (;;) {
 509                 if (is->is_reconnecting == false &&
 510                     is->is_terminating == false &&
 511                     (STAILQ_EMPTY(&is->is_postponed) ||
 512                      ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn)))
 513                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 514
 515                 /* Terminate supersedes reconnect. */
 516                 if (is->is_terminating) {
 517                         ISCSI_SESSION_UNLOCK(is);
 518                         iscsi_maintenance_thread_terminate(is);
 519                         kthread_exit();
 520                         return;
 521                 }
 522
 523                 if (is->is_reconnecting) {
 524                         ISCSI_SESSION_UNLOCK(is);
 525                         iscsi_maintenance_thread_reconnect(is);
 526                         ISCSI_SESSION_LOCK(is);
 527                         continue;
 528                 }
 529
 530                 iscsi_session_send_postponed(is);
 531         }
 532         ISCSI_SESSION_UNLOCK(is);
 533 }
 534
 535 static void
 536 iscsi_session_reconnect(struct iscsi_session *is)
 537 {
 538
 539         /*
 540          * XXX: We can't use locking here, because
 541          *      it's being called from various contexts.
 542          *      Hope it doesn't break anything.
 543          */
 544         if (is->is_reconnecting)
 545                 return;
 546
 547         is->is_reconnecting = true;
 548         cv_signal(&is->is_maintenance_cv);
 549 }
 550
 551 static void
 552 iscsi_session_terminate(struct iscsi_session *is)
 553 {
 554
 555         if (is->is_terminating)
 556                 return;
 557
 558         is->is_terminating = true;
 559
 560 #if 0
 561         iscsi_session_logout(is);
 562 #endif
 563         cv_signal(&is->is_maintenance_cv);
 564 }
 565
 566 static void
 567 iscsi_callout(void *context)
 568 {
 569         struct icl_pdu *request;
 570         struct iscsi_bhs_nop_out *bhsno;
 571         struct iscsi_session *is;
 572         bool reconnect_needed = false;
 573         sbintime_t sbt, pr;
 574
 575         is = context;
 576
 577         ISCSI_SESSION_LOCK(is);
 578         if (is->is_terminating) {
 579                 ISCSI_SESSION_UNLOCK(is);
 580                 return;
 581         }
 582
 583         sbt = mstosbt(995);
 584         pr  = mstosbt(10);
 585         callout_schedule_sbt(&is->is_callout, sbt, pr, 0);
 586
 587         if (is->is_conf.isc_enable == 0)
 588                 goto out;
 589
 590         is->is_timeout++;
 591
 592         if (is->is_waiting_for_iscsid) {
 593                 if (iscsid_timeout > 0 && is->is_timeout > iscsid_timeout) {
 594                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 595                             "for %d seconds; reconnecting",
 596                             is->is_timeout);
 597                         reconnect_needed = true;
 598                 }
 599                 goto out;
 600         }
 601
 602         if (is->is_login_phase) {
 603                 if (is->is_login_timeout > 0 && is->is_timeout > is->is_login_timeout) {
 604                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 605                             "reconnecting", is->is_timeout);
 606                         reconnect_needed = true;
 607                 }
 608                 goto out;
 609         }
 610
 611         if (is->is_ping_timeout <= 0) {
 612                 /*
 613                  * Pings are disabled.  Don't send NOP-Out in this case.
 614                  * Reset the timeout, to avoid triggering reconnection,
 615                  * should the user decide to reenable them.
 616                  */
 617                 is->is_timeout = 0;
 618                 goto out;
 619         }
 620
 621         if (is->is_timeout >= is->is_ping_timeout) {
 622                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 623                     "reconnecting", is->is_ping_timeout);
 624                 reconnect_needed = true;
 625                 goto out;
 626         }
 627
 628         ISCSI_SESSION_UNLOCK(is);
 629
 630         /*
 631          * If the ping was reset less than one second ago - which means
 632          * that we've received some PDU during the last second - assume
 633          * the traffic flows correctly and don't bother sending a NOP-Out.
 634          *
 635          * (It's 2 - one for one second, and one for incrementing is_timeout
 636          * earlier in this routine.)
 637          */
 638         if (is->is_timeout < 2)
 639                 return;
 640
 641         request = icl_pdu_new(is->is_conn, M_NOWAIT);
 642         if (request == NULL) {
 643                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 644                 return;
 645         }
 646         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 647         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 648             ISCSI_BHS_OPCODE_IMMEDIATE;
 649         bhsno->bhsno_flags = 0x80;
 650         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 651         iscsi_pdu_queue(request);
 652         return;
 653
 654 out:
 655         if (is->is_terminating) {
 656                 ISCSI_SESSION_UNLOCK(is);
 657                 return;
 658         }
 659
 660         ISCSI_SESSION_UNLOCK(is);
 661
 662         if (reconnect_needed)
 663                 iscsi_session_reconnect(is);
 664 }
 665
 666 static void
 667 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 668 {
 669         const struct iscsi_bhs_data_in *bhsdi;
 670         struct iscsi_session *is;
 671         uint32_t expcmdsn, maxcmdsn, statsn;
 672
 673         is = PDU_SESSION(response);
 674
 675         ISCSI_SESSION_LOCK_ASSERT(is);
 676
 677         /*
 678          * We're only using fields common for all the response
 679          * (target -> initiator) PDUs.
 680          */
 681         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 682         /*
 683          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 684          * and Residual Count only have meaningful content if the S bit
 685          * is set to 1", so we also need to check the bit specific for
 686          * Data-In PDU.
 687          */
 688         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 689             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 690                 statsn = ntohl(bhsdi->bhsdi_statsn);
 691                 if (statsn != is->is_statsn && statsn != (is->is_statsn + 1)) {
 692                         /* XXX: This is normal situation for MCS */
 693                         ISCSI_SESSION_WARN(is, "PDU 0x%x StatSN %u != "
 694                             "session ExpStatSN %u (or + 1); reconnecting",
 695                             bhsdi->bhsdi_opcode, statsn, is->is_statsn);
 696                         iscsi_session_reconnect(is);
 697                 }
 698                 if (ISCSI_SNGT(statsn, is->is_statsn))
 699                         is->is_statsn = statsn;
 700         }
 701
 702         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 703         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 704
 705         if (ISCSI_SNLT(maxcmdsn + 1, expcmdsn)) {
 706                 ISCSI_SESSION_DEBUG(is,
 707                     "PDU MaxCmdSN %u + 1 < PDU ExpCmdSN %u; ignoring",
 708                     maxcmdsn, expcmdsn);
 709         } else {
 710                 if (ISCSI_SNGT(maxcmdsn, is->is_maxcmdsn)) {
 711                         is->is_maxcmdsn = maxcmdsn;
 712
 713                         /*
 714                          * Command window increased; kick the maintanance thread
 715                          * to send out postponed commands.
 716                          */
 717                         if (!STAILQ_EMPTY(&is->is_postponed))
 718                                 cv_signal(&is->is_maintenance_cv);
 719                 } else if (ISCSI_SNLT(maxcmdsn, is->is_maxcmdsn)) {
 720                         /* XXX: This is normal situation for MCS */
 721                         ISCSI_SESSION_DEBUG(is,
 722                             "PDU MaxCmdSN %u < session MaxCmdSN %u; ignoring",
 723                             maxcmdsn, is->is_maxcmdsn);
 724                 }
 725
 726                 if (ISCSI_SNGT(expcmdsn, is->is_expcmdsn)) {
 727                         is->is_expcmdsn = expcmdsn;
 728                 } else if (ISCSI_SNLT(expcmdsn, is->is_expcmdsn)) {
 729                         /* XXX: This is normal situation for MCS */
 730                         ISCSI_SESSION_DEBUG(is,
 731                             "PDU ExpCmdSN %u < session ExpCmdSN %u; ignoring",
 732                             expcmdsn, is->is_expcmdsn);
 733                 }
 734         }
 735
 736         /*
 737          * Every incoming PDU - not just NOP-In - resets the ping timer.
 738          * The purpose of the timeout is to reset the connection when it stalls;
 739          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 740          * in some queue.
 741          */
 742         is->is_timeout = 0;
 743 }
 744
 745 static void
 746 iscsi_receive_callback(struct icl_pdu *response)
 747 {
 748         struct iscsi_session *is;
 749
 750         is = PDU_SESSION(response);
 751
 752         ISCSI_SESSION_LOCK(is);
 753
 754         iscsi_pdu_update_statsn(response);
 755
 756 #ifdef ICL_KERNEL_PROXY
 757         if (is->is_login_phase) {
 758                 if (is->is_login_pdu == NULL)
 759                         is->is_login_pdu = response;
 760                 else
 761                         icl_pdu_free(response);
 762                 ISCSI_SESSION_UNLOCK(is);
 763                 cv_signal(&is->is_login_cv);
 764                 return;
 765         }
 766 #endif
 767
 768         /*
 769          * The handling routine is responsible for freeing the PDU
 770          * when it's no longer needed.
 771          */
 772         switch (response->ip_bhs->bhs_opcode) {
 773         case ISCSI_BHS_OPCODE_NOP_IN:
 774                 iscsi_pdu_handle_nop_in(response);
 775                 ISCSI_SESSION_UNLOCK(is);
 776                 break;
 777         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 778                 iscsi_pdu_handle_scsi_response(response);
 779                 /* Session lock dropped inside. */
 780                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 781                 break;
 782         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
 783                 iscsi_pdu_handle_task_response(response);
 784                 ISCSI_SESSION_UNLOCK(is);
 785                 break;
 786         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 787                 iscsi_pdu_handle_data_in(response);
 788                 /* Session lock dropped inside. */
 789                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
 790                 break;
 791         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 792                 iscsi_pdu_handle_logout_response(response);
 793                 ISCSI_SESSION_UNLOCK(is);
 794                 break;
 795         case ISCSI_BHS_OPCODE_R2T:
 796                 iscsi_pdu_handle_r2t(response);
 797                 ISCSI_SESSION_UNLOCK(is);
 798                 break;
 799         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 800                 iscsi_pdu_handle_async_message(response);
 801                 ISCSI_SESSION_UNLOCK(is);
 802                 break;
 803         case ISCSI_BHS_OPCODE_REJECT:
 804                 iscsi_pdu_handle_reject(response);
 805                 ISCSI_SESSION_UNLOCK(is);
 806                 break;
 807         default:
 808                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 809                     "opcode 0x%x; reconnecting",
 810                     response->ip_bhs->bhs_opcode);
 811                 iscsi_session_reconnect(is);
 812                 ISCSI_SESSION_UNLOCK(is);
 813                 icl_pdu_free(response);
 814         }
 815 }
 816
 817 static void
 818 iscsi_error_callback(struct icl_conn *ic)
 819 {
 820         struct iscsi_session *is;
 821
 822         is = CONN_SESSION(ic);
 823
 824         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 825         iscsi_session_reconnect(is);
 826 }
 827
 828 static void
 829 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 830 {
 831         struct iscsi_session *is;
 832         struct iscsi_bhs_nop_out *bhsno;
 833         struct iscsi_bhs_nop_in *bhsni;
 834         struct icl_pdu *request;
 835         void *data = NULL;
 836         size_t datasize;
 837         int error;
 838
 839         is = PDU_SESSION(response);
 840         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 841
 842         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 843                 /*
 844                  * Nothing to do; iscsi_pdu_update_statsn() already
 845                  * zeroed the timeout.
 846                  */
 847                 icl_pdu_free(response);
 848                 return;
 849         }
 850
 851         datasize = icl_pdu_data_segment_length(response);
 852         if (datasize > 0) {
 853                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 854                 if (data == NULL) {
 855                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 856                             "reconnecting");
 857                         icl_pdu_free(response);
 858                         iscsi_session_reconnect(is);
 859                         return;
 860                 }
 861                 icl_pdu_get_data(response, 0, data, datasize);
 862         }
 863
 864         request = icl_pdu_new(response->ip_conn, M_NOWAIT);
 865         if (request == NULL) {
 866                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 867                     "reconnecting");
 868                 free(data, M_ISCSI);
 869                 icl_pdu_free(response);
 870                 iscsi_session_reconnect(is);
 871                 return;
 872         }
 873         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 874         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 875             ISCSI_BHS_OPCODE_IMMEDIATE;
 876         bhsno->bhsno_flags = 0x80;
 877         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 878         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 879         if (datasize > 0) {
 880                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 881                 if (error != 0) {
 882                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 883                             "reconnecting");
 884                         free(data, M_ISCSI);
 885                         icl_pdu_free(request);
 886                         icl_pdu_free(response);
 887                         iscsi_session_reconnect(is);
 888                         return;
 889                 }
 890                 free(data, M_ISCSI);
 891         }
 892
 893         icl_pdu_free(response);
 894         iscsi_pdu_queue_locked(request);
 895 }
 896
 897 static void
 898 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 899 {
 900         struct iscsi_bhs_scsi_response *bhssr;
 901         struct iscsi_outstanding *io;
 902         struct iscsi_session *is;
 903         union ccb *ccb;
 904         struct ccb_scsiio *csio;
 905         size_t data_segment_len, received;
 906         uint16_t sense_len;
 907         uint32_t resid;
 908
 909         is = PDU_SESSION(response);
 910
 911         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 912         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 913         if (io == NULL || io->io_ccb == NULL) {
 914                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 915                 icl_pdu_free(response);
 916                 iscsi_session_reconnect(is);
 917                 ISCSI_SESSION_UNLOCK(is);
 918                 return;
 919         }
 920
 921         ccb = io->io_ccb;
 922         if (bhssr->bhssr_response == BHSSR_RESPONSE_COMMAND_COMPLETED) {
 923                 if (ntohl(bhssr->bhssr_expdatasn) != io->io_datasn) {
 924                         ISCSI_SESSION_WARN(is,
 925                             "ExpDataSN mismatch in SCSI Response (%u vs %u)",
 926                             ntohl(bhssr->bhssr_expdatasn), io->io_datasn);
 927
 928                         /*
 929                          * XXX: Permit an ExpDataSN of zero for errors.
 930                          *
 931                          * This doesn't conform to RFC 7143, but some
 932                          * targets seem to do this.
 933                          */
 934                         if (bhssr->bhssr_status != 0 &&
 935                             bhssr->bhssr_expdatasn == htonl(0))
 936                                 goto skip_expdatasn;
 937
 938                         icl_pdu_free(response);
 939                         iscsi_session_reconnect(is);
 940                         ISCSI_SESSION_UNLOCK(is);
 941                         return;
 942                 }
 943         } else {
 944                 if (bhssr->bhssr_expdatasn != htonl(0)) {
 945                         ISCSI_SESSION_WARN(is,
 946                             "ExpDataSN mismatch in SCSI Response (%u vs 0)",
 947                             ntohl(bhssr->bhssr_expdatasn));
 948                         icl_pdu_free(response);
 949                         iscsi_session_reconnect(is);
 950                         ISCSI_SESSION_UNLOCK(is);
 951                         return;
 952                 }
 953         }
 954 skip_expdatasn:
 955
 956         /*
 957          * With iSER, after getting good response we can be sure
 958          * that all the data has been successfully transferred.
 959          */
 960         if (is->is_conn->ic_iser) {
 961                 resid = ntohl(bhssr->bhssr_residual_count);
 962                 if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW) {
 963                         io->io_received = ccb->csio.dxfer_len - resid;
 964                 } else if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
 965                         ISCSI_SESSION_WARN(is, "overflow: target indicates %d", resid);
 966                 } else {
 967                         io->io_received = ccb->csio.dxfer_len;
 968                 }
 969         }
 970
 971         received = io->io_received;
 972         iscsi_outstanding_remove(is, io);
 973         ISCSI_SESSION_UNLOCK(is);
 974
 975         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 976                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 977                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 978                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 979                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 980                 }
 981                 ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 982         } else if (bhssr->bhssr_status == 0) {
 983                 ccb->ccb_h.status = CAM_REQ_CMP;
 984         } else {
 985                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 986                         xpt_freeze_devq(ccb->ccb_h.path, 1);
 987                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 988                 }
 989                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 990                 ccb->csio.scsi_status = bhssr->bhssr_status;
 991         }
 992
 993         csio = &ccb->csio;
 994         data_segment_len = icl_pdu_data_segment_length(response);
 995         if (data_segment_len > 0) {
 996                 if (data_segment_len < sizeof(sense_len)) {
 997                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 998                             data_segment_len);
 999                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1000                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
1001                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
1002                         }
1003                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
1004                         goto out;
1005                 }
1006                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
1007                 sense_len = ntohs(sense_len);
1008 #if 0
1009                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
1010                     sense_len, data_segment_len);
1011 #endif
1012                 if (sizeof(sense_len) + sense_len > data_segment_len) {
1013                         ISCSI_SESSION_WARN(is, "truncated data segment "
1014                             "(%zd bytes, should be %zd)",
1015                             data_segment_len, sizeof(sense_len) + sense_len);
1016                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1017                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
1018                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
1019                         }
1020                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
1021                         goto out;
1022                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
1023                         ISCSI_SESSION_WARN(is, "oversize data segment "
1024                             "(%zd bytes, should be %zd)",
1025                             data_segment_len, sizeof(sense_len) + sense_len);
1026                 if (sense_len > csio->sense_len) {
1027                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
1028                             sense_len, csio->sense_len);
1029                         sense_len = csio->sense_len;
1030                 }
1031                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
1032                 csio->sense_resid = csio->sense_len - sense_len;
1033                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
1034         }
1035
1036 out:
1037         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
1038                 csio->resid = ntohl(bhssr->bhssr_residual_count);
1039
1040         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1041                 KASSERT(received <= csio->dxfer_len,
1042                     ("received > csio->dxfer_len"));
1043                 if (received < csio->dxfer_len) {
1044                         if (csio->resid != csio->dxfer_len - received) {
1045                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1046                                     "target indicates %d, we calculated %zd",
1047                                     csio->resid, csio->dxfer_len - received);
1048                         }
1049                         csio->resid = csio->dxfer_len - received;
1050                 }
1051         }
1052
1053         xpt_done(ccb);
1054         icl_pdu_free(response);
1055 }
1056
1057 static void
1058 iscsi_pdu_handle_task_response(struct icl_pdu *response)
1059 {
1060         struct iscsi_bhs_task_management_response *bhstmr;
1061         struct iscsi_outstanding *io, *aio;
1062         struct iscsi_session *is;
1063
1064         is = PDU_SESSION(response);
1065
1066         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
1067         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
1068         if (io == NULL || io->io_ccb != NULL) {
1069                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
1070                     bhstmr->bhstmr_initiator_task_tag);
1071                 icl_pdu_free(response);
1072                 iscsi_session_reconnect(is);
1073                 return;
1074         }
1075
1076         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
1077                 ISCSI_SESSION_WARN(is, "task response 0x%x",
1078                     bhstmr->bhstmr_response);
1079         } else {
1080                 aio = iscsi_outstanding_find(is, io->io_referenced_task_tag);
1081                 if (aio != NULL && aio->io_ccb != NULL)
1082                         iscsi_session_terminate_task(is, aio, CAM_REQ_ABORTED);
1083         }
1084
1085         iscsi_outstanding_remove(is, io);
1086         icl_pdu_free(response);
1087 }
1088
1089 static void
1090 iscsi_pdu_get_data_csio(struct icl_pdu *response, size_t pdu_offset,
1091     struct ccb_scsiio *csio, size_t oreceived, size_t data_segment_len)
1092 {
1093         switch (csio->ccb_h.flags & CAM_DATA_MASK) {
1094         case CAM_DATA_BIO:
1095                 icl_pdu_get_bio(response, pdu_offset,
1096                     (struct bio *)csio->data_ptr, oreceived, data_segment_len);
1097                 break;
1098         case CAM_DATA_VADDR:
1099                 icl_pdu_get_data(response, pdu_offset,
1100                     csio->data_ptr + oreceived, data_segment_len);
1101                 break;
1102         default:
1103                 __assert_unreachable();
1104         }
1105 }
1106
1107 static void
1108 iscsi_pdu_handle_data_in(struct icl_pdu *response)
1109 {
1110         struct iscsi_bhs_data_in *bhsdi;
1111         struct iscsi_outstanding *io;
1112         struct iscsi_session *is;
1113         union ccb *ccb;
1114         struct ccb_scsiio *csio;
1115         size_t data_segment_len, received, oreceived;
1116
1117         is = PDU_SESSION(response);
1118         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
1119         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
1120         if (io == NULL || io->io_ccb == NULL) {
1121                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
1122                 icl_pdu_free(response);
1123                 iscsi_session_reconnect(is);
1124                 ISCSI_SESSION_UNLOCK(is);
1125                 return;
1126         }
1127
1128         if (io->io_datasn != ntohl(bhsdi->bhsdi_datasn)) {
1129                 ISCSI_SESSION_WARN(is, "received Data-In PDU with "
1130                     "DataSN %u, while expected %u; dropping connection",
1131                     ntohl(bhsdi->bhsdi_datasn), io->io_datasn);
1132                 icl_pdu_free(response);
1133                 iscsi_session_reconnect(is);
1134                 ISCSI_SESSION_UNLOCK(is);
1135                 return;
1136         }
1137         io->io_datasn += response->ip_additional_pdus + 1;
1138
1139         data_segment_len = icl_pdu_data_segment_length(response);
1140         if (data_segment_len == 0) {
1141                 /*
1142                  * "The sending of 0 length data segments should be avoided,
1143                  * but initiators and targets MUST be able to properly receive
1144                  * 0 length data segments."
1145                  */
1146                 ISCSI_SESSION_UNLOCK(is);
1147                 icl_pdu_free(response);
1148                 return;
1149         }
1150
1151         /*
1152          * We need to track this for security reasons - without it, malicious target
1153          * could respond to SCSI READ without sending Data-In PDUs, which would result
1154          * in read operation on the initiator side returning random kernel data.
1155          */
1156         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1157                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1158                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1159                 icl_pdu_free(response);
1160                 iscsi_session_reconnect(is);
1161                 ISCSI_SESSION_UNLOCK(is);
1162                 return;
1163         }
1164
1165         ccb = io->io_ccb;
1166         csio = &ccb->csio;
1167
1168         if (io->io_received + data_segment_len > csio->dxfer_len) {
1169                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1170                     "at offset %zd, buffer is %d)",
1171                     data_segment_len, io->io_received, csio->dxfer_len);
1172                 icl_pdu_free(response);
1173                 iscsi_session_reconnect(is);
1174                 ISCSI_SESSION_UNLOCK(is);
1175                 return;
1176         }
1177
1178         oreceived = io->io_received;
1179         io->io_received += data_segment_len;
1180         received = io->io_received;
1181         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0)
1182                 iscsi_outstanding_remove(is, io);
1183         ISCSI_SESSION_UNLOCK(is);
1184
1185         iscsi_pdu_get_data_csio(response, 0, csio, oreceived, data_segment_len);
1186
1187         /*
1188          * XXX: Check F.
1189          */
1190         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1191                 /*
1192                  * Nothing more to do.
1193                  */
1194                 icl_pdu_free(response);
1195                 return;
1196         }
1197
1198         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1199         if (bhsdi->bhsdi_status == 0) {
1200                 ccb->ccb_h.status = CAM_REQ_CMP;
1201         } else {
1202                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1203                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1204                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1205                 }
1206                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1207                 csio->scsi_status = bhsdi->bhsdi_status;
1208         }
1209
1210         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1211                 KASSERT(received <= csio->dxfer_len,
1212                     ("received > csio->dxfer_len"));
1213                 if (received < csio->dxfer_len) {
1214                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1215                         if (csio->resid != csio->dxfer_len - received) {
1216                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1217                                     "target indicates %d, we calculated %zd",
1218                                     csio->resid, csio->dxfer_len - received);
1219                         }
1220                         csio->resid = csio->dxfer_len - received;
1221                 }
1222         }
1223
1224         xpt_done(ccb);
1225         icl_pdu_free(response);
1226 }
1227
1228 static void
1229 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1230 {
1231
1232         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1233         icl_pdu_free(response);
1234 }
1235
1236 static int
1237 iscsi_pdu_append_data_csio(struct icl_pdu *request, struct ccb_scsiio *csio,
1238     size_t off, size_t len, int how)
1239 {
1240         switch (csio->ccb_h.flags & CAM_DATA_MASK) {
1241         case CAM_DATA_BIO:
1242                 return (icl_pdu_append_bio(request,
1243                         (struct bio *)csio->data_ptr, off, len, how));
1244         case CAM_DATA_VADDR:
1245                 return (icl_pdu_append_data(request, csio->data_ptr + off, len,
1246                     how));
1247         default:
1248                 __assert_unreachable();
1249         }
1250 }
1251
1252 static void
1253 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1254 {
1255         struct icl_pdu *request;
1256         struct iscsi_session *is;
1257         struct iscsi_bhs_r2t *bhsr2t;
1258         struct iscsi_bhs_data_out *bhsdo;
1259         struct iscsi_outstanding *io;
1260         struct ccb_scsiio *csio;
1261         size_t off, len, max_send_data_segment_length, total_len;
1262         int error;
1263         uint32_t datasn = 0;
1264
1265         is = PDU_SESSION(response);
1266
1267         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1268         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1269         if (io == NULL || io->io_ccb == NULL) {
1270                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1271                     bhsr2t->bhsr2t_initiator_task_tag);
1272                 icl_pdu_free(response);
1273                 iscsi_session_reconnect(is);
1274                 return;
1275         }
1276
1277         csio = &io->io_ccb->csio;
1278
1279         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1280                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1281                 icl_pdu_free(response);
1282                 iscsi_session_reconnect(is);
1283                 return;
1284         }
1285
1286         /*
1287          * XXX: Verify R2TSN.
1288          */
1289
1290         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1291         if (off > csio->dxfer_len) {
1292                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1293                     "%zd, buffer is %d; reconnecting", off, csio->dxfer_len);
1294                 icl_pdu_free(response);
1295                 iscsi_session_reconnect(is);
1296                 return;
1297         }
1298
1299         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1300         if (total_len == 0 || total_len > csio->dxfer_len) {
1301                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1302                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1303                 icl_pdu_free(response);
1304                 iscsi_session_reconnect(is);
1305                 return;
1306         }
1307
1308         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1309
1310         if (is->is_conn->ic_hw_isomax != 0)
1311                 max_send_data_segment_length = is->is_conn->ic_hw_isomax;
1312         else
1313                 max_send_data_segment_length =
1314                     is->is_conn->ic_max_send_data_segment_length;
1315         for (;;) {
1316                 len = total_len;
1317
1318                 if (len > max_send_data_segment_length)
1319                         len = max_send_data_segment_length;
1320
1321                 if (off + len > csio->dxfer_len) {
1322                         ISCSI_SESSION_WARN(is, "target requested invalid "
1323                             "length/offset %zd, buffer is %d; reconnecting",
1324                             off + len, csio->dxfer_len);
1325                         icl_pdu_free(response);
1326                         iscsi_session_reconnect(is);
1327                         return;
1328                 }
1329
1330                 request = icl_pdu_new(response->ip_conn, M_NOWAIT);
1331                 if (request == NULL) {
1332                         icl_pdu_free(response);
1333                         iscsi_session_reconnect(is);
1334                         return;
1335                 }
1336
1337                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1338                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1339                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1340                 bhsdo->bhsdo_initiator_task_tag =
1341                     bhsr2t->bhsr2t_initiator_task_tag;
1342                 bhsdo->bhsdo_target_transfer_tag =
1343                     bhsr2t->bhsr2t_target_transfer_tag;
1344                 bhsdo->bhsdo_datasn = htonl(datasn);
1345                 bhsdo->bhsdo_buffer_offset = htonl(off);
1346                 error = iscsi_pdu_append_data_csio(request, csio, off, len,
1347                     M_NOWAIT | ICL_NOCOPY);
1348                 if (error != 0) {
1349                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1350                             "reconnecting");
1351                         icl_pdu_free(request);
1352                         icl_pdu_free(response);
1353                         iscsi_session_reconnect(is);
1354                         return;
1355                 }
1356
1357                 datasn += howmany(len,
1358                     is->is_conn->ic_max_send_data_segment_length);
1359                 off += len;
1360                 total_len -= len;
1361
1362                 if (total_len == 0) {
1363                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1364                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1365                 } else {
1366                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1367                 }
1368
1369                 iscsi_pdu_queue_locked(request);
1370
1371                 if (total_len == 0)
1372                         break;
1373         }
1374
1375         icl_pdu_free(response);
1376 }
1377
1378 static void
1379 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1380 {
1381         struct iscsi_bhs_asynchronous_message *bhsam;
1382         struct iscsi_session *is;
1383
1384         is = PDU_SESSION(response);
1385         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1386         switch (bhsam->bhsam_async_event) {
1387         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1388                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1389                 iscsi_session_logout(is);
1390                 iscsi_session_terminate(is);
1391                 break;
1392         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1393                 ISCSI_SESSION_WARN(is, "target indicates it will drop the connection");
1394                 break;
1395         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1396                 ISCSI_SESSION_WARN(is, "target indicates it will drop the session");
1397                 break;
1398         default:
1399                 /*
1400                  * XXX: Technically, we're obligated to also handle
1401                  *      parameter renegotiation.
1402                  */
1403                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1404                 break;
1405         }
1406
1407         icl_pdu_free(response);
1408 }
1409
1410 static void
1411 iscsi_pdu_handle_reject(struct icl_pdu *response)
1412 {
1413         struct iscsi_bhs_reject *bhsr;
1414         struct iscsi_session *is;
1415
1416         is = PDU_SESSION(response);
1417         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1418         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1419             bhsr->bhsr_reason);
1420
1421         icl_pdu_free(response);
1422 }
1423
1424 static int
1425 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1426     struct iscsi_daemon_request *request)
1427 {
1428         struct iscsi_session *is;
1429         struct icl_drv_limits idl;
1430         int error;
1431
1432         sx_slock(&sc->sc_lock);
1433         for (;;) {
1434                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1435                         ISCSI_SESSION_LOCK(is);
1436                         if (is->is_conf.isc_enable == 0 &&
1437                             is->is_conf.isc_discovery == 0) {
1438                                 ISCSI_SESSION_UNLOCK(is);
1439                                 continue;
1440                         }
1441                         if (is->is_waiting_for_iscsid)
1442                                 break;
1443                         ISCSI_SESSION_UNLOCK(is);
1444                 }
1445
1446                 if (is == NULL) {
1447                         if (sc->sc_unloading) {
1448                                 sx_sunlock(&sc->sc_lock);
1449                                 return (ENXIO);
1450                         }
1451
1452                         /*
1453                          * No session requires attention from iscsid(8); wait.
1454                          */
1455                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1456                         if (error != 0) {
1457                                 sx_sunlock(&sc->sc_lock);
1458                                 return (error);
1459                         }
1460                         continue;
1461                 }
1462
1463                 is->is_waiting_for_iscsid = false;
1464                 is->is_login_phase = true;
1465                 is->is_reason[0] = '\0';
1466                 ISCSI_SESSION_UNLOCK(is);
1467
1468                 request->idr_session_id = is->is_id;
1469                 memcpy(&request->idr_isid, &is->is_isid,
1470                     sizeof(request->idr_isid));
1471                 request->idr_tsih = 0;  /* New or reinstated session. */
1472                 memcpy(&request->idr_conf, &is->is_conf,
1473                     sizeof(request->idr_conf));
1474
1475                 error = icl_limits(is->is_conf.isc_offload,
1476                     is->is_conf.isc_iser, &idl);
1477                 if (error != 0) {
1478                         ISCSI_SESSION_WARN(is, "icl_limits for offload \"%s\" "
1479                             "failed with error %d", is->is_conf.isc_offload,
1480                             error);
1481                         sx_sunlock(&sc->sc_lock);
1482                         return (error);
1483                 }
1484                 request->idr_limits.isl_max_recv_data_segment_length =
1485                     idl.idl_max_recv_data_segment_length;
1486                 request->idr_limits.isl_max_send_data_segment_length =
1487                     idl.idl_max_send_data_segment_length;
1488                 request->idr_limits.isl_max_burst_length =
1489                     idl.idl_max_burst_length;
1490                 request->idr_limits.isl_first_burst_length =
1491                     idl.idl_first_burst_length;
1492
1493                 sx_sunlock(&sc->sc_lock);
1494                 return (0);
1495         }
1496 }
1497
1498 static int
1499 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1500     struct iscsi_daemon_handoff *handoff)
1501 {
1502         struct iscsi_session *is;
1503         struct icl_conn *ic;
1504         int error;
1505
1506         sx_slock(&sc->sc_lock);
1507
1508         /*
1509          * Find the session to hand off socket to.
1510          */
1511         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1512                 if (is->is_id == handoff->idh_session_id)
1513                         break;
1514         }
1515         if (is == NULL) {
1516                 sx_sunlock(&sc->sc_lock);
1517                 return (ESRCH);
1518         }
1519         ISCSI_SESSION_LOCK(is);
1520         ic = is->is_conn;
1521         if (is->is_conf.isc_discovery || is->is_terminating) {
1522                 ISCSI_SESSION_UNLOCK(is);
1523                 sx_sunlock(&sc->sc_lock);
1524                 return (EINVAL);
1525         }
1526         if (is->is_connected) {
1527                 /*
1528                  * This might have happened because another iscsid(8)
1529                  * instance handed off the connection in the meantime.
1530                  * Just return.
1531                  */
1532                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1533                     "session");
1534                 ISCSI_SESSION_UNLOCK(is);
1535                 sx_sunlock(&sc->sc_lock);
1536                 return (EBUSY);
1537         }
1538
1539         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1540             sizeof(is->is_target_alias));
1541         is->is_tsih = handoff->idh_tsih;
1542         is->is_statsn = handoff->idh_statsn;
1543         is->is_protocol_level = handoff->idh_protocol_level;
1544         is->is_initial_r2t = handoff->idh_initial_r2t;
1545         is->is_immediate_data = handoff->idh_immediate_data;
1546
1547         ic->ic_max_recv_data_segment_length =
1548             handoff->idh_max_recv_data_segment_length;
1549         ic->ic_max_send_data_segment_length =
1550             handoff->idh_max_send_data_segment_length;
1551         is->is_max_burst_length = handoff->idh_max_burst_length;
1552         is->is_first_burst_length = handoff->idh_first_burst_length;
1553
1554         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1555                 ic->ic_header_crc32c = true;
1556         else
1557                 ic->ic_header_crc32c = false;
1558         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1559                 ic->ic_data_crc32c = true;
1560         else
1561                 ic->ic_data_crc32c = false;
1562         ic->ic_maxtags = maxtags;
1563
1564         is->is_cmdsn = 0;
1565         is->is_expcmdsn = 0;
1566         is->is_maxcmdsn = 0;
1567         is->is_waiting_for_iscsid = false;
1568         is->is_login_phase = false;
1569         is->is_timeout = 0;
1570         is->is_ping_timeout = is->is_conf.isc_ping_timeout;
1571         if (is->is_ping_timeout < 0)
1572                 is->is_ping_timeout = ping_timeout;
1573         is->is_login_timeout = is->is_conf.isc_login_timeout;
1574         if (is->is_login_timeout < 0)
1575                 is->is_login_timeout = login_timeout;
1576         is->is_connected = true;
1577         is->is_reason[0] = '\0';
1578
1579         ISCSI_SESSION_UNLOCK(is);
1580
1581         /*
1582          * If we're going through the proxy, the idh_socket will be 0,
1583          * and the ICL module can simply ignore this call.  It can also
1584          * use it to determine it's no longer in the Login phase.
1585          */
1586         error = icl_conn_handoff(ic, handoff->idh_socket);
1587         if (error != 0) {
1588                 sx_sunlock(&sc->sc_lock);
1589                 iscsi_session_terminate(is);
1590                 return (error);
1591         }
1592
1593         sx_sunlock(&sc->sc_lock);
1594
1595         if (is->is_sim != NULL) {
1596                 /*
1597                  * When reconnecting, there already is SIM allocated for the session.
1598                  */
1599                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1600                 ISCSI_SESSION_LOCK(is);
1601                 ISCSI_SESSION_DEBUG(is, "releasing");
1602                 is->is_simq_frozen = false;
1603                 xpt_release_simq(is->is_sim, 1);
1604                 ISCSI_SESSION_UNLOCK(is);
1605
1606         } else {
1607                 ISCSI_SESSION_LOCK(is);
1608                 is->is_devq = cam_simq_alloc(ic->ic_maxtags);
1609                 if (is->is_devq == NULL) {
1610                         ISCSI_SESSION_UNLOCK(is);
1611                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1612                         iscsi_session_terminate(is);
1613                         return (ENOMEM);
1614                 }
1615
1616                 is->is_sim = cam_sim_alloc(iscsi_action, NULL, "iscsi",
1617                     is, is->is_id /* unit */, &is->is_lock,
1618                     1, ic->ic_maxtags, is->is_devq);
1619                 if (is->is_sim == NULL) {
1620                         ISCSI_SESSION_UNLOCK(is);
1621                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1622                         cam_simq_free(is->is_devq);
1623                         iscsi_session_terminate(is);
1624                         return (ENOMEM);
1625                 }
1626
1627                 if (xpt_bus_register(is->is_sim, NULL, 0) != 0) {
1628                         ISCSI_SESSION_UNLOCK(is);
1629                         ISCSI_SESSION_WARN(is, "failed to register bus");
1630                         iscsi_session_terminate(is);
1631                         return (ENOMEM);
1632                 }
1633
1634                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1635                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1636                     CAM_LUN_WILDCARD);
1637                 if (error != CAM_REQ_CMP) {
1638                         ISCSI_SESSION_UNLOCK(is);
1639                         ISCSI_SESSION_WARN(is, "failed to create path");
1640                         iscsi_session_terminate(is);
1641                         return (ENOMEM);
1642                 }
1643                 ISCSI_SESSION_UNLOCK(is);
1644         }
1645
1646         return (0);
1647 }
1648
1649 static int
1650 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1651     struct iscsi_daemon_fail *fail)
1652 {
1653         struct iscsi_session *is;
1654
1655         sx_slock(&sc->sc_lock);
1656
1657         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1658                 if (is->is_id == fail->idf_session_id)
1659                         break;
1660         }
1661         if (is == NULL) {
1662                 sx_sunlock(&sc->sc_lock);
1663                 return (ESRCH);
1664         }
1665         ISCSI_SESSION_LOCK(is);
1666         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1667             fail->idf_reason);
1668         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1669         //is->is_waiting_for_iscsid = false;
1670         //is->is_login_phase = true;
1671         //iscsi_session_reconnect(is);
1672         ISCSI_SESSION_UNLOCK(is);
1673         sx_sunlock(&sc->sc_lock);
1674
1675         return (0);
1676 }
1677
1678 #ifdef ICL_KERNEL_PROXY
1679 static int
1680 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1681     struct iscsi_daemon_connect *idc)
1682 {
1683         struct iscsi_session *is;
1684         struct sockaddr *from_sa, *to_sa;
1685         int error;
1686
1687         sx_slock(&sc->sc_lock);
1688         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1689                 if (is->is_id == idc->idc_session_id)
1690                         break;
1691         }
1692         if (is == NULL) {
1693                 sx_sunlock(&sc->sc_lock);
1694                 return (ESRCH);
1695         }
1696         sx_sunlock(&sc->sc_lock);
1697
1698         if (idc->idc_from_addrlen > 0) {
1699                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1700                 if (error != 0) {
1701                         ISCSI_SESSION_WARN(is,
1702                             "getsockaddr failed with error %d", error);
1703                         return (error);
1704                 }
1705         } else {
1706                 from_sa = NULL;
1707         }
1708         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1709         if (error != 0) {
1710                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1711                     error);
1712                 free(from_sa, M_SONAME);
1713                 return (error);
1714         }
1715
1716         ISCSI_SESSION_LOCK(is);
1717         is->is_statsn = 0;
1718         is->is_cmdsn = 0;
1719         is->is_expcmdsn = 0;
1720         is->is_maxcmdsn = 0;
1721         is->is_waiting_for_iscsid = false;
1722         is->is_login_phase = true;
1723         is->is_timeout = 0;
1724         ISCSI_SESSION_UNLOCK(is);
1725
1726         error = icl_conn_connect(is->is_conn, idc->idc_domain,
1727             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1728         free(from_sa, M_SONAME);
1729         free(to_sa, M_SONAME);
1730
1731         /*
1732          * Digests are always disabled during login phase.
1733          */
1734         is->is_conn->ic_header_crc32c = false;
1735         is->is_conn->ic_data_crc32c = false;
1736
1737         return (error);
1738 }
1739
1740 static int
1741 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1742     struct iscsi_daemon_send *ids)
1743 {
1744         struct iscsi_session *is;
1745         struct icl_pdu *ip;
1746         size_t datalen;
1747         void *data;
1748         int error;
1749
1750         sx_slock(&sc->sc_lock);
1751         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1752                 if (is->is_id == ids->ids_session_id)
1753                         break;
1754         }
1755         if (is == NULL) {
1756                 sx_sunlock(&sc->sc_lock);
1757                 return (ESRCH);
1758         }
1759         sx_sunlock(&sc->sc_lock);
1760
1761         if (is->is_login_phase == false)
1762                 return (EBUSY);
1763
1764         if (is->is_terminating || is->is_reconnecting)
1765                 return (EIO);
1766
1767         datalen = ids->ids_data_segment_len;
1768         if (datalen > is->is_conn->ic_max_send_data_segment_length)
1769                 return (EINVAL);
1770         if (datalen > 0) {
1771                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1772                 error = copyin(ids->ids_data_segment, data, datalen);
1773                 if (error != 0) {
1774                         free(data, M_ISCSI);
1775                         return (error);
1776                 }
1777         }
1778
1779         ip = icl_pdu_new(is->is_conn, M_WAITOK);
1780         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1781         if (datalen > 0) {
1782                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1783                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1784                 free(data, M_ISCSI);
1785         }
1786         iscsi_pdu_queue(ip);
1787
1788         return (0);
1789 }
1790
1791 static int
1792 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1793     struct iscsi_daemon_receive *idr)
1794 {
1795         struct iscsi_session *is;
1796         struct icl_pdu *ip;
1797         void *data;
1798         int error;
1799
1800         sx_slock(&sc->sc_lock);
1801         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1802                 if (is->is_id == idr->idr_session_id)
1803                         break;
1804         }
1805         if (is == NULL) {
1806                 sx_sunlock(&sc->sc_lock);
1807                 return (ESRCH);
1808         }
1809         sx_sunlock(&sc->sc_lock);
1810
1811         if (is->is_login_phase == false)
1812                 return (EBUSY);
1813
1814         ISCSI_SESSION_LOCK(is);
1815         while (is->is_login_pdu == NULL &&
1816             is->is_terminating == false &&
1817             is->is_reconnecting == false) {
1818                 error = cv_wait_sig(&is->is_login_cv, &is->is_lock);
1819                 if (error != 0) {
1820                         ISCSI_SESSION_UNLOCK(is);
1821                         return (error);
1822                 }
1823         }
1824         if (is->is_terminating || is->is_reconnecting) {
1825                 ISCSI_SESSION_UNLOCK(is);
1826                 return (EIO);
1827         }
1828         ip = is->is_login_pdu;
1829         is->is_login_pdu = NULL;
1830         ISCSI_SESSION_UNLOCK(is);
1831
1832         if (ip->ip_data_len > idr->idr_data_segment_len) {
1833                 icl_pdu_free(ip);
1834                 return (EMSGSIZE);
1835         }
1836
1837         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1838         if (ip->ip_data_len > 0) {
1839                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1840                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1841                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1842                 free(data, M_ISCSI);
1843         }
1844
1845         icl_pdu_free(ip);
1846
1847         return (0);
1848 }
1849 #endif /* ICL_KERNEL_PROXY */
1850
1851 static void
1852 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1853 {
1854         /*
1855          * Just make sure all the fields are null-terminated.
1856          *
1857          * XXX: This is not particularly secure.  We should
1858          *      create our own conf and then copy in relevant
1859          *      fields.
1860          */
1861         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1862         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1863         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1864         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1865         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1866         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1867         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1868         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1869         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1870 }
1871
1872 static bool
1873 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1874 {
1875
1876         if (isc->isc_initiator[0] == '\0') {
1877                 ISCSI_DEBUG("empty isc_initiator");
1878                 return (false);
1879         }
1880
1881         if (isc->isc_target_addr[0] == '\0') {
1882                 ISCSI_DEBUG("empty isc_target_addr");
1883                 return (false);
1884         }
1885
1886         if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1887                 ISCSI_DEBUG("non-empty isc_target for discovery session");
1888                 return (false);
1889         }
1890
1891         if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1892                 ISCSI_DEBUG("empty isc_target for non-discovery session");
1893                 return (false);
1894         }
1895
1896         return (true);
1897 }
1898
1899 static int
1900 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1901 {
1902         struct iscsi_session *is;
1903         const struct iscsi_session *is2;
1904         int error;
1905         sbintime_t sbt, pr;
1906
1907         iscsi_sanitize_session_conf(&isa->isa_conf);
1908         if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1909                 return (EINVAL);
1910
1911         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1912         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1913
1914         sx_xlock(&sc->sc_lock);
1915
1916         /*
1917          * Prevent duplicates.
1918          */
1919         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1920                 if (!!is->is_conf.isc_discovery !=
1921                     !!is2->is_conf.isc_discovery)
1922                         continue;
1923
1924                 if (strcmp(is->is_conf.isc_target_addr,
1925                     is2->is_conf.isc_target_addr) != 0)
1926                         continue;
1927
1928                 if (is->is_conf.isc_discovery == 0 &&
1929                     strcmp(is->is_conf.isc_target,
1930                     is2->is_conf.isc_target) != 0)
1931                         continue;
1932
1933                 sx_xunlock(&sc->sc_lock);
1934                 free(is, M_ISCSI);
1935                 return (EBUSY);
1936         }
1937
1938         is->is_conn = icl_new_conn(is->is_conf.isc_offload,
1939             is->is_conf.isc_iser, "iscsi", &is->is_lock);
1940         if (is->is_conn == NULL) {
1941                 sx_xunlock(&sc->sc_lock);
1942                 free(is, M_ISCSI);
1943                 return (EINVAL);
1944         }
1945         is->is_conn->ic_receive = iscsi_receive_callback;
1946         is->is_conn->ic_error = iscsi_error_callback;
1947         is->is_conn->ic_prv0 = is;
1948         TAILQ_INIT(&is->is_outstanding);
1949         STAILQ_INIT(&is->is_postponed);
1950         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1951         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1952 #ifdef ICL_KERNEL_PROXY
1953         cv_init(&is->is_login_cv, "iscsi_login");
1954 #endif
1955
1956         /*
1957          * Set some default values, from RFC 3720, section 12.
1958          *
1959          * These values are updated by the handoff IOCTL, but are
1960          * needed prior to the handoff to support sending the ISER
1961          * login PDU.
1962          */
1963         is->is_conn->ic_max_recv_data_segment_length = 8192;
1964         is->is_conn->ic_max_send_data_segment_length = 8192;
1965         is->is_max_burst_length = 262144;
1966         is->is_first_burst_length = 65536;
1967
1968         is->is_softc = sc;
1969         sc->sc_last_session_id++;
1970         is->is_id = sc->sc_last_session_id;
1971         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1972         arc4rand(&is->is_isid[1], 5, 0);
1973         is->is_tsih = 0;
1974         callout_init(&is->is_callout, 1);
1975
1976         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1977         if (error != 0) {
1978                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1979                 sx_xunlock(&sc->sc_lock);
1980                 return (error);
1981         }
1982         is->is_ping_timeout = is->is_conf.isc_ping_timeout;
1983         if (is->is_ping_timeout < 0)
1984                 is->is_ping_timeout = ping_timeout;
1985         is->is_login_timeout = is->is_conf.isc_login_timeout;
1986         if (is->is_login_timeout < 0)
1987                 is->is_login_timeout = login_timeout;
1988
1989         sbt = mstosbt(995);
1990         pr = mstosbt(10);
1991         callout_reset_sbt(&is->is_callout, sbt, pr, iscsi_callout, is, 0);
1992         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1993
1994         ISCSI_SESSION_LOCK(is);
1995         /*
1996          * Don't notify iscsid(8) if the session is disabled and it's not
1997          * a discovery session,
1998          */
1999         if (is->is_conf.isc_enable == 0 && is->is_conf.isc_discovery == 0) {
2000                 ISCSI_SESSION_UNLOCK(is);
2001                 sx_xunlock(&sc->sc_lock);
2002                 return (0);
2003         }
2004
2005         is->is_waiting_for_iscsid = true;
2006         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
2007         ISCSI_SESSION_UNLOCK(is);
2008         cv_signal(&sc->sc_cv);
2009         sx_xunlock(&sc->sc_lock);
2010         return (0);
2011 }
2012
2013 static bool
2014 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
2015     unsigned int id2, const struct iscsi_session_conf *c2)
2016 {
2017
2018         if (id2 != 0 && id2 != id1)
2019                 return (false);
2020         if (c2->isc_target[0] != '\0' &&
2021             strcmp(c1->isc_target, c2->isc_target) != 0)
2022                 return (false);
2023         if (c2->isc_target_addr[0] != '\0' &&
2024             strcmp(c1->isc_target_addr, c2->isc_target_addr) != 0)
2025                 return (false);
2026         return (true);
2027 }
2028
2029 static int
2030 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
2031     struct iscsi_session_remove *isr)
2032 {
2033         struct iscsi_session *is, *tmp;
2034         bool found = false;
2035
2036         iscsi_sanitize_session_conf(&isr->isr_conf);
2037
2038         sx_xlock(&sc->sc_lock);
2039         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
2040                 ISCSI_SESSION_LOCK(is);
2041                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
2042                     isr->isr_session_id, &isr->isr_conf)) {
2043                         found = true;
2044                         iscsi_session_logout(is);
2045                         iscsi_session_terminate(is);
2046                 }
2047                 ISCSI_SESSION_UNLOCK(is);
2048         }
2049         sx_xunlock(&sc->sc_lock);
2050
2051         if (!found)
2052                 return (ESRCH);
2053
2054         return (0);
2055 }
2056
2057 static int
2058 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
2059 {
2060         int error;
2061         unsigned int i = 0;
2062         struct iscsi_session *is;
2063         struct iscsi_session_state iss;
2064
2065         sx_slock(&sc->sc_lock);
2066         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
2067                 if (i >= isl->isl_nentries) {
2068                         sx_sunlock(&sc->sc_lock);
2069                         return (EMSGSIZE);
2070                 }
2071                 memset(&iss, 0, sizeof(iss));
2072                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
2073                 iss.iss_id = is->is_id;
2074                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
2075                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
2076                 strlcpy(iss.iss_offload, is->is_conn->ic_offload, sizeof(iss.iss_offload));
2077
2078                 if (is->is_conn->ic_header_crc32c)
2079                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
2080                 else
2081                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
2082
2083                 if (is->is_conn->ic_data_crc32c)
2084                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
2085                 else
2086                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
2087
2088                 iss.iss_max_send_data_segment_length =
2089                     is->is_conn->ic_max_send_data_segment_length;
2090                 iss.iss_max_recv_data_segment_length =
2091                     is->is_conn->ic_max_recv_data_segment_length;
2092                 iss.iss_max_burst_length = is->is_max_burst_length;
2093                 iss.iss_first_burst_length = is->is_first_burst_length;
2094                 iss.iss_immediate_data = is->is_immediate_data;
2095                 iss.iss_connected = is->is_connected;
2096
2097                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
2098                 if (error != 0) {
2099                         sx_sunlock(&sc->sc_lock);
2100                         return (error);
2101                 }
2102                 i++;
2103         }
2104         sx_sunlock(&sc->sc_lock);
2105
2106         isl->isl_nentries = i;
2107
2108         return (0);
2109 }
2110
2111 static int
2112 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
2113     struct iscsi_session_modify *ism)
2114 {
2115         struct iscsi_session *is;
2116         const struct iscsi_session *is2;
2117
2118         iscsi_sanitize_session_conf(&ism->ism_conf);
2119         if (iscsi_valid_session_conf(&ism->ism_conf) == false)
2120                 return (EINVAL);
2121
2122         sx_xlock(&sc->sc_lock);
2123         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
2124                 ISCSI_SESSION_LOCK(is);
2125                 if (is->is_id == ism->ism_session_id) {
2126                         /* Note that the session remains locked. */
2127                         break;
2128                 }
2129                 ISCSI_SESSION_UNLOCK(is);
2130         }
2131         if (is == NULL) {
2132                 sx_xunlock(&sc->sc_lock);
2133                 return (ESRCH);
2134         }
2135
2136         /*
2137          * Prevent duplicates.
2138          */
2139         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
2140                 if (is == is2)
2141                         continue;
2142
2143                 if (!!ism->ism_conf.isc_discovery !=
2144                     !!is2->is_conf.isc_discovery)
2145                         continue;
2146
2147                 if (strcmp(ism->ism_conf.isc_target_addr,
2148                     is2->is_conf.isc_target_addr) != 0)
2149                         continue;
2150
2151                 if (ism->ism_conf.isc_discovery == 0 &&
2152                     strcmp(ism->ism_conf.isc_target,
2153                     is2->is_conf.isc_target) != 0)
2154                         continue;
2155
2156                 ISCSI_SESSION_UNLOCK(is);
2157                 sx_xunlock(&sc->sc_lock);
2158                 return (EBUSY);
2159         }
2160
2161         sx_xunlock(&sc->sc_lock);
2162
2163         memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
2164         ISCSI_SESSION_UNLOCK(is);
2165
2166         iscsi_session_reconnect(is);
2167
2168         return (0);
2169 }
2170
2171 static int
2172 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
2173     struct thread *td)
2174 {
2175         struct iscsi_softc *sc;
2176
2177         sc = dev->si_drv1;
2178
2179         switch (cmd) {
2180         case ISCSIDWAIT:
2181                 return (iscsi_ioctl_daemon_wait(sc,
2182                     (struct iscsi_daemon_request *)arg));
2183         case ISCSIDHANDOFF:
2184                 return (iscsi_ioctl_daemon_handoff(sc,
2185                     (struct iscsi_daemon_handoff *)arg));
2186         case ISCSIDFAIL:
2187                 return (iscsi_ioctl_daemon_fail(sc,
2188                     (struct iscsi_daemon_fail *)arg));
2189 #ifdef ICL_KERNEL_PROXY
2190         case ISCSIDCONNECT:
2191                 return (iscsi_ioctl_daemon_connect(sc,
2192                     (struct iscsi_daemon_connect *)arg));
2193         case ISCSIDSEND:
2194                 return (iscsi_ioctl_daemon_send(sc,
2195                     (struct iscsi_daemon_send *)arg));
2196         case ISCSIDRECEIVE:
2197                 return (iscsi_ioctl_daemon_receive(sc,
2198                     (struct iscsi_daemon_receive *)arg));
2199 #endif /* ICL_KERNEL_PROXY */
2200         case ISCSISADD:
2201                 return (iscsi_ioctl_session_add(sc,
2202                     (struct iscsi_session_add *)arg));
2203         case ISCSISREMOVE:
2204                 return (iscsi_ioctl_session_remove(sc,
2205                     (struct iscsi_session_remove *)arg));
2206         case ISCSISLIST:
2207                 return (iscsi_ioctl_session_list(sc,
2208                     (struct iscsi_session_list *)arg));
2209         case ISCSISMODIFY:
2210                 return (iscsi_ioctl_session_modify(sc,
2211                     (struct iscsi_session_modify *)arg));
2212         default:
2213                 return (EINVAL);
2214         }
2215 }
2216
2217 static struct iscsi_outstanding *
2218 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
2219 {
2220         struct iscsi_outstanding *io;
2221
2222         ISCSI_SESSION_LOCK_ASSERT(is);
2223
2224         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2225                 if (io->io_initiator_task_tag == initiator_task_tag)
2226                         return (io);
2227         }
2228         return (NULL);
2229 }
2230
2231 static struct iscsi_outstanding *
2232 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
2233 {
2234         struct iscsi_outstanding *io;
2235
2236         ISCSI_SESSION_LOCK_ASSERT(is);
2237
2238         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2239                 if (io->io_ccb == ccb)
2240                         return (io);
2241         }
2242         return (NULL);
2243 }
2244
2245 static struct iscsi_outstanding *
2246 iscsi_outstanding_add(struct iscsi_session *is, struct icl_pdu *request,
2247     union ccb *ccb, uint32_t *initiator_task_tagp)
2248 {
2249         struct iscsi_outstanding *io;
2250         int error;
2251
2252         ISCSI_SESSION_LOCK_ASSERT(is);
2253
2254         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
2255         if (io == NULL) {
2256                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes",
2257                     sizeof(*io));
2258                 return (NULL);
2259         }
2260
2261         error = icl_conn_task_setup(is->is_conn, request, &ccb->csio,
2262             initiator_task_tagp, &io->io_icl_prv);
2263         if (error != 0) {
2264                 ISCSI_SESSION_WARN(is,
2265                     "icl_conn_task_setup() failed with error %d", error);
2266                 uma_zfree(iscsi_outstanding_zone, io);
2267                 return (NULL);
2268         }
2269
2270         KASSERT(iscsi_outstanding_find(is, *initiator_task_tagp) == NULL,
2271             ("initiator_task_tag 0x%x already added", *initiator_task_tagp));
2272
2273         io->io_initiator_task_tag = *initiator_task_tagp;
2274         io->io_ccb = ccb;
2275         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
2276         return (io);
2277 }
2278
2279 static void
2280 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
2281 {
2282
2283         ISCSI_SESSION_LOCK_ASSERT(is);
2284
2285         icl_conn_task_done(is->is_conn, io->io_icl_prv);
2286         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
2287         uma_zfree(iscsi_outstanding_zone, io);
2288 }
2289
2290 static void
2291 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
2292 {
2293         struct icl_pdu *request;
2294         struct iscsi_bhs_task_management_request *bhstmr;
2295         struct ccb_abort *cab = &ccb->cab;
2296         struct iscsi_outstanding *io, *aio;
2297         uint32_t initiator_task_tag;
2298
2299         ISCSI_SESSION_LOCK_ASSERT(is);
2300
2301 #if 0
2302         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2303 #else
2304         if (is->is_login_phase) {
2305                 ccb->ccb_h.status = CAM_REQ_ABORTED;
2306                 xpt_done(ccb);
2307                 return;
2308         }
2309 #endif
2310
2311         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
2312         if (aio == NULL) {
2313                 ccb->ccb_h.status = CAM_REQ_CMP;
2314                 xpt_done(ccb);
2315                 return;
2316         }
2317
2318         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2319         if (request == NULL) {
2320                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2321                 xpt_done(ccb);
2322                 return;
2323         }
2324
2325         initiator_task_tag = is->is_initiator_task_tag++;
2326         if (initiator_task_tag == 0xffffffff)
2327                 initiator_task_tag = is->is_initiator_task_tag++;
2328
2329         io = iscsi_outstanding_add(is, request, NULL, &initiator_task_tag);
2330         if (io == NULL) {
2331                 icl_pdu_free(request);
2332                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2333                 xpt_done(ccb);
2334                 return;
2335         }
2336         io->io_referenced_task_tag = aio->io_initiator_task_tag;
2337
2338         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2339         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
2340         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
2341         bhstmr->bhstmr_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
2342         bhstmr->bhstmr_initiator_task_tag = initiator_task_tag;
2343         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2344
2345         iscsi_pdu_queue_locked(request);
2346 }
2347
2348 static void
2349 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2350 {
2351         struct icl_pdu *request;
2352         struct iscsi_bhs_scsi_command *bhssc;
2353         struct ccb_scsiio *csio;
2354         struct iscsi_outstanding *io;
2355         size_t len;
2356         uint32_t initiator_task_tag;
2357         int error;
2358
2359         ISCSI_SESSION_LOCK_ASSERT(is);
2360
2361 #if 0
2362         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2363 #else
2364         if (is->is_login_phase) {
2365                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2366                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2367                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2368                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2369                 }
2370                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2371                 xpt_done(ccb);
2372                 return;
2373         }
2374 #endif
2375
2376         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2377         if (request == NULL) {
2378                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2379                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2380                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2381                 }
2382                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2383                 xpt_done(ccb);
2384                 return;
2385         }
2386
2387         initiator_task_tag = is->is_initiator_task_tag++;
2388         if (initiator_task_tag == 0xffffffff)
2389                 initiator_task_tag = is->is_initiator_task_tag++;
2390
2391         io = iscsi_outstanding_add(is, request, ccb, &initiator_task_tag);
2392         if (io == NULL) {
2393                 icl_pdu_free(request);
2394                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2395                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2396                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2397                 }
2398                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2399                 xpt_done(ccb);
2400                 return;
2401         }
2402
2403         csio = &ccb->csio;
2404         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2405         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2406         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2407         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2408         case CAM_DIR_IN:
2409                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2410                 break;
2411         case CAM_DIR_OUT:
2412                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2413                 break;
2414         }
2415
2416         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2417                 switch (csio->tag_action) {
2418                 case MSG_HEAD_OF_Q_TAG:
2419                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2420                         break;
2421                 case MSG_ORDERED_Q_TAG:
2422                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2423                         break;
2424                 case MSG_ACA_TASK:
2425                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2426                         break;
2427                 case MSG_SIMPLE_Q_TAG:
2428                 default:
2429                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2430                         break;
2431                 }
2432         } else
2433                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2434
2435         if (is->is_protocol_level >= 2) {
2436                 bhssc->bhssc_pri = (csio->priority << BHSSC_PRI_SHIFT) &
2437                     BHSSC_PRI_MASK;
2438         }
2439
2440         bhssc->bhssc_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
2441         bhssc->bhssc_initiator_task_tag = initiator_task_tag;
2442         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2443         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2444             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2445
2446         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2447                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2448         else
2449                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2450
2451         if (is->is_immediate_data &&
2452             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2453                 len = csio->dxfer_len;
2454                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2455                 if (len > is->is_first_burst_length) {
2456                         ISCSI_SESSION_DEBUG(is, "len %zd -> %d", len, is->is_first_burst_length);
2457                         len = is->is_first_burst_length;
2458                 }
2459                 if (len > is->is_conn->ic_max_send_data_segment_length) {
2460                         ISCSI_SESSION_DEBUG(is, "len %zd -> %d", len,
2461                             is->is_conn->ic_max_send_data_segment_length);
2462                         len = is->is_conn->ic_max_send_data_segment_length;
2463                 }
2464
2465                 error = iscsi_pdu_append_data_csio(request, csio, 0, len,
2466                     M_NOWAIT | ICL_NOCOPY);
2467                 if (error != 0) {
2468                         iscsi_outstanding_remove(is, io);
2469                         icl_pdu_free(request);
2470                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2471                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2472                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2473                         }
2474                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2475                         xpt_done(ccb);
2476                         return;
2477                 }
2478         }
2479         iscsi_pdu_queue_locked(request);
2480 }
2481
2482 static void
2483 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2484 {
2485         struct iscsi_session *is;
2486
2487         is = cam_sim_softc(sim);
2488
2489         ISCSI_SESSION_LOCK_ASSERT(is);
2490
2491         if (is->is_terminating ||
2492             (is->is_connected == false && fail_on_disconnection)) {
2493                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2494                 xpt_done(ccb);
2495                 return;
2496         }
2497
2498         /*
2499          * Make sure CAM doesn't sneak in a CCB just after freezing the queue.
2500          */
2501         if (is->is_simq_frozen == true) {
2502                 ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
2503                 ccb->ccb_h.status |= CAM_REQUEUE_REQ;
2504                 /* Don't freeze the devq - the SIM queue is already frozen. */
2505                 xpt_done(ccb);
2506                 return;
2507         }
2508
2509         switch (ccb->ccb_h.func_code) {
2510         case XPT_PATH_INQ:
2511         {
2512                 struct ccb_pathinq *cpi = &ccb->cpi;
2513
2514                 cpi->version_num = 1;
2515                 cpi->hba_inquiry = PI_TAG_ABLE;
2516                 cpi->target_sprt = 0;
2517                 cpi->hba_misc = PIM_EXTLUNS;
2518                 /*
2519                  * XXX: It shouldn't ever be NULL; this could be turned
2520                  *      into a KASSERT eventually.
2521                  */
2522                 if (is->is_conn == NULL)
2523                         ISCSI_WARN("NULL conn");
2524                 else if (is->is_conn->ic_unmapped)
2525                         cpi->hba_misc |= PIM_UNMAPPED;
2526                 cpi->hba_eng_cnt = 0;
2527                 cpi->max_target = 0;
2528                 /*
2529                  * Note that the variable below is only relevant for targets
2530                  * that don't claim compliance with anything above SPC2, which
2531                  * means they don't support REPORT_LUNS.
2532                  */
2533                 cpi->max_lun = 255;
2534                 cpi->initiator_id = ~0;
2535                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2536                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2537                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2538                 cpi->unit_number = cam_sim_unit(sim);
2539                 cpi->bus_id = cam_sim_bus(sim);
2540                 cpi->base_transfer_speed = 150000; /* XXX */
2541                 cpi->transport = XPORT_ISCSI;
2542                 cpi->transport_version = 0;
2543                 cpi->protocol = PROTO_SCSI;
2544                 cpi->protocol_version = SCSI_REV_SPC3;
2545                 cpi->maxio = maxphys;
2546                 cpi->ccb_h.status = CAM_REQ_CMP;
2547                 break;
2548         }
2549         case XPT_GET_TRAN_SETTINGS:
2550         {
2551                 struct ccb_trans_settings       *cts;
2552                 struct ccb_trans_settings_scsi  *scsi;
2553
2554                 cts = &ccb->cts;
2555                 scsi = &cts->proto_specific.scsi;
2556
2557                 cts->protocol = PROTO_SCSI;
2558                 cts->protocol_version = SCSI_REV_SPC3;
2559                 cts->transport = XPORT_ISCSI;
2560                 cts->transport_version = 0;
2561                 scsi->valid = CTS_SCSI_VALID_TQ;
2562                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2563                 cts->ccb_h.status = CAM_REQ_CMP;
2564                 break;
2565         }
2566         case XPT_CALC_GEOMETRY:
2567                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2568                 ccb->ccb_h.status = CAM_REQ_CMP;
2569                 break;
2570 #if 0
2571         /*
2572          * XXX: What's the point?
2573          */
2574         case XPT_RESET_BUS:
2575         case XPT_TERM_IO:
2576                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2577                 ccb->ccb_h.status = CAM_REQ_CMP;
2578                 break;
2579 #endif
2580         case XPT_ABORT:
2581                 iscsi_action_abort(is, ccb);
2582                 return;
2583         case XPT_SCSI_IO:
2584                 iscsi_action_scsiio(is, ccb);
2585                 return;
2586         default:
2587 #if 0
2588                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2589 #endif
2590                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2591                 break;
2592         }
2593         xpt_done(ccb);
2594 }
2595
2596 static void
2597 iscsi_terminate_sessions(struct iscsi_softc *sc)
2598 {
2599         struct iscsi_session *is;
2600
2601         sx_slock(&sc->sc_lock);
2602         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2603                 iscsi_session_terminate(is);
2604         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2605                 ISCSI_DEBUG("waiting for sessions to terminate");
2606                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2607         }
2608         ISCSI_DEBUG("all sessions terminated");
2609         sx_sunlock(&sc->sc_lock);
2610 }
2611
2612 static void
2613 iscsi_shutdown_pre(struct iscsi_softc *sc)
2614 {
2615         struct iscsi_session *is;
2616
2617         if (!fail_on_shutdown)
2618                 return;
2619
2620         /*
2621          * If we have any sessions waiting for reconnection, request
2622          * maintenance thread to fail them immediately instead of waiting
2623          * for reconnect timeout.
2624          *
2625          * This prevents LUNs with mounted filesystems that are supported
2626          * by disconnected iSCSI sessions from hanging, however it will
2627          * fail all queued BIOs.
2628          */
2629         ISCSI_DEBUG("forcing failing all disconnected sessions due to shutdown");
2630
2631         fail_on_disconnection = 1;
2632
2633         sx_slock(&sc->sc_lock);
2634         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
2635                 ISCSI_SESSION_LOCK(is);
2636                 if (!is->is_connected) {
2637                         ISCSI_SESSION_DEBUG(is, "force failing disconnected session early");
2638                         iscsi_session_reconnect(is);
2639                 }
2640                 ISCSI_SESSION_UNLOCK(is);
2641         }
2642         sx_sunlock(&sc->sc_lock);
2643 }
2644
2645 static void
2646 iscsi_shutdown_post(struct iscsi_softc *sc)
2647 {
2648
2649         if (!KERNEL_PANICKED()) {
2650                 ISCSI_DEBUG("removing all sessions due to shutdown");
2651                 iscsi_terminate_sessions(sc);
2652         }
2653 }
2654
2655 static int
2656 iscsi_load(void)
2657 {
2658         int error;
2659
2660         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2661         sx_init(&sc->sc_lock, "iscsi");
2662         TAILQ_INIT(&sc->sc_sessions);
2663         cv_init(&sc->sc_cv, "iscsi_cv");
2664
2665         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2666             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2667             UMA_ALIGN_PTR, 0);
2668
2669         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2670             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2671         if (error != 0) {
2672                 ISCSI_WARN("failed to create device node, error %d", error);
2673                 return (error);
2674         }
2675         sc->sc_cdev->si_drv1 = sc;
2676
2677         sc->sc_shutdown_pre_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
2678             iscsi_shutdown_pre, sc, SHUTDOWN_PRI_FIRST);
2679         /*
2680          * shutdown_post_sync needs to run after filesystem shutdown and before
2681          * CAM shutdown - otherwise when rebooting with an iSCSI session that is
2682          * disconnected but has outstanding requests, dashutdown() will hang on
2683          * cam_periph_runccb().
2684          */
2685         sc->sc_shutdown_post_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2686             iscsi_shutdown_post, sc, SHUTDOWN_PRI_DEFAULT - 1);
2687
2688         return (0);
2689 }
2690
2691 static int
2692 iscsi_unload(void)
2693 {
2694
2695         /* Awaken any threads asleep in iscsi_ioctl(). */
2696         sx_xlock(&sc->sc_lock);
2697         sc->sc_unloading = true;
2698         cv_signal(&sc->sc_cv);
2699         sx_xunlock(&sc->sc_lock);
2700
2701         if (sc->sc_cdev != NULL) {
2702                 ISCSI_DEBUG("removing device node");
2703                 destroy_dev(sc->sc_cdev);
2704                 ISCSI_DEBUG("device node removed");
2705         }
2706
2707         if (sc->sc_shutdown_pre_eh != NULL)
2708                 EVENTHANDLER_DEREGISTER(shutdown_pre_sync, sc->sc_shutdown_pre_eh);
2709         if (sc->sc_shutdown_post_eh != NULL)
2710                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_post_eh);
2711
2712         iscsi_terminate_sessions(sc);
2713
2714         uma_zdestroy(iscsi_outstanding_zone);
2715         sx_destroy(&sc->sc_lock);
2716         cv_destroy(&sc->sc_cv);
2717         free(sc, M_ISCSI);
2718         return (0);
2719 }
2720
2721 static int
2722 iscsi_quiesce(void)
2723 {
2724         sx_slock(&sc->sc_lock);
2725         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2726                 sx_sunlock(&sc->sc_lock);
2727                 return (EBUSY);
2728         }
2729         sx_sunlock(&sc->sc_lock);
2730         return (0);
2731 }
2732
2733 static int
2734 iscsi_modevent(module_t mod, int what, void *arg)
2735 {
2736         int error;
2737
2738         switch (what) {
2739         case MOD_LOAD:
2740                 error = iscsi_load();
2741                 break;
2742         case MOD_UNLOAD:
2743                 error = iscsi_unload();
2744                 break;
2745         case MOD_QUIESCE:
2746                 error = iscsi_quiesce();
2747                 break;
2748         default:
2749                 error = EINVAL;
2750                 break;
2751         }
2752         return (error);
2753 }
2754
2755 moduledata_t iscsi_data = {
2756         "iscsi",
2757         iscsi_modevent,
2758         0
2759 };
2760
2761 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2762 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2763 MODULE_DEPEND(iscsi, icl, 1, 1, 1);