sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #include <sys/param.h>
  33 #include <sys/condvar.h>
  34 #include <sys/conf.h>
  35 #include <sys/endian.h>
  36 #include <sys/eventhandler.h>
  37 #include <sys/file.h>
  38 #include <sys/kernel.h>
  39 #include <sys/kthread.h>
  40 #include <sys/lock.h>
  41 #include <sys/malloc.h>
  42 #include <sys/mutex.h>
  43 #include <sys/module.h>
  44 #include <sys/sysctl.h>
  45 #include <sys/systm.h>
  46 #include <sys/sx.h>
  47 #include <vm/uma.h>
  48
  49 #include <cam/cam.h>
  50 #include <cam/cam_ccb.h>
  51 #include <cam/cam_xpt.h>
  52 #include <cam/cam_debug.h>
  53 #include <cam/cam_sim.h>
  54 #include <cam/cam_xpt_sim.h>
  55 #include <cam/cam_xpt_periph.h>
  56 #include <cam/cam_periph.h>
  57 #include <cam/scsi/scsi_all.h>
  58 #include <cam/scsi/scsi_message.h>
  59
  60 #include "iscsi_ioctl.h"
  61 #include "iscsi.h"
  62 #include "icl.h"
  63 #include "iscsi_proto.h"
  64
  65 #ifdef ICL_KERNEL_PROXY
  66 #include <sys/socketvar.h>
  67 #endif
  68
  69 #ifdef ICL_KERNEL_PROXY
  70 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
  71 #endif
  72
  73 /*
  74  * XXX: This is global so the iscsi_unload() can access it.
  75  *      Think about how to do this properly.
  76  */
  77 static struct iscsi_softc       *sc;
  78
  79 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  80 static int debug = 1;
  81 TUNABLE_INT("kern.iscsi.debug", &debug);
  82 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  83     &debug, 0, "Enable debug messages");
  84 static int ping_timeout = 5;
  85 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
  86 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
  87     0, "Timeout for ping (NOP-Out) requests, in seconds");
  88 static int iscsid_timeout = 60;
  89 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
  90 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
  91     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  92 static int login_timeout = 60;
  93 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
  94 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
  95     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  96 static int maxtags = 255;
  97 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
  98 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
  99     0, "Max number of IO requests queued");
 100
 101 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
 102 static uma_zone_t iscsi_outstanding_zone;
 103
 104 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 105 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 106
 107 #define ISCSI_DEBUG(X, ...)                                             \
 108         do {                                                            \
 109                 if (debug > 1)                                          \
 110                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 111         } while (0)
 112
 113 #define ISCSI_WARN(X, ...)                                              \
 114         do {                                                            \
 115                 if (debug > 0) {                                        \
 116                         printf("WARNING: %s: " X "\n",                  \
 117                             __func__, ## __VA_ARGS__);                  \
 118                 }                                                       \
 119         } while (0)
 120
 121 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
 122         do {                                                            \
 123                 if (debug > 1) {                                        \
 124                         printf("%s: %s (%s): " X "\n",                  \
 125                             __func__, S->is_conf.isc_target_addr,       \
 126                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 127                 }                                                       \
 128         } while (0)
 129
 130 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
 131         do {                                                            \
 132                 if (debug > 0) {                                        \
 133                         printf("WARNING: %s (%s): " X "\n",             \
 134                             S->is_conf.isc_target_addr,                 \
 135                             S->is_conf.isc_target, ## __VA_ARGS__);     \
 136                 }                                                       \
 137         } while (0)
 138
 139 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 140 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 141 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 142
 143 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 144                     int mode, struct thread *td);
 145
 146 static struct cdevsw iscsi_cdevsw = {
 147      .d_version = D_VERSION,
 148      .d_ioctl   = iscsi_ioctl,
 149      .d_name    = "iscsi",
 150 };
 151
 152 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 153 static void     iscsi_pdu_queue(struct icl_pdu *request);
 154 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 155 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 156 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 157 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 158 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 159 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 160 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 161 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 162 static void     iscsi_session_reconnect(struct iscsi_session *is);
 163 static void     iscsi_session_terminate(struct iscsi_session *is);
 164 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 165 static void     iscsi_poll(struct cam_sim *sim);
 166 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 167                     uint32_t initiator_task_tag);
 168 static int      iscsi_outstanding_add(struct iscsi_session *is,
 169                     uint32_t initiator_task_tag, union ccb *ccb);
 170 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 171                     struct iscsi_outstanding *io);
 172
 173 static bool
 174 iscsi_pdu_prepare(struct icl_pdu *request)
 175 {
 176         struct iscsi_session *is;
 177         struct iscsi_bhs_scsi_command *bhssc;
 178
 179         is = PDU_SESSION(request);
 180
 181         ISCSI_SESSION_LOCK_ASSERT(is);
 182
 183         /*
 184          * We're only using fields common for all the request
 185          * (initiator -> target) PDUs.
 186          */
 187         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 188
 189         /*
 190          * Data-Out PDU does not contain CmdSN.
 191          */
 192         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 193                 if (is->is_cmdsn > is->is_maxcmdsn &&
 194                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 195                         /*
 196                          * Current MaxCmdSN prevents us from sending any more
 197                          * SCSI Command PDUs to the target; postpone the PDU.
 198                          * It will get resent by either iscsi_pdu_queue(),
 199                          * or by maintenance thread.
 200                          */
 201 #if 0
 202                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %d, ExpCmdSN %d, MaxCmdSN %d, opcode 0x%x",
 203                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn, bhssc->bhssc_opcode);
 204 #endif
 205                         return (true);
 206                 }
 207                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 208                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 209                         is->is_cmdsn++;
 210         }
 211         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 212
 213         return (false);
 214 }
 215
 216 static void
 217 iscsi_session_send_postponed(struct iscsi_session *is)
 218 {
 219         struct icl_pdu *request;
 220         bool postpone;
 221
 222         ISCSI_SESSION_LOCK_ASSERT(is);
 223
 224         while (!STAILQ_EMPTY(&is->is_postponed)) {
 225                 request = STAILQ_FIRST(&is->is_postponed);
 226                 postpone = iscsi_pdu_prepare(request);
 227                 if (postpone)
 228                         break;
 229                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 230                 icl_pdu_queue(request);
 231         }
 232 }
 233
 234 static void
 235 iscsi_pdu_queue_locked(struct icl_pdu *request)
 236 {
 237         struct iscsi_session *is;
 238         bool postpone;
 239
 240         is = PDU_SESSION(request);
 241         ISCSI_SESSION_LOCK_ASSERT(is);
 242         iscsi_session_send_postponed(is);
 243         postpone = iscsi_pdu_prepare(request);
 244         if (postpone) {
 245                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 246                 return;
 247         }
 248         icl_pdu_queue(request);
 249 }
 250
 251 static void
 252 iscsi_pdu_queue(struct icl_pdu *request)
 253 {
 254         struct iscsi_session *is;
 255
 256         is = PDU_SESSION(request);
 257         ISCSI_SESSION_LOCK(is);
 258         iscsi_pdu_queue_locked(request);
 259         ISCSI_SESSION_UNLOCK(is);
 260 }
 261
 262 static void
 263 iscsi_session_logout(struct iscsi_session *is)
 264 {
 265         struct icl_pdu *request;
 266         struct iscsi_bhs_logout_request *bhslr;
 267
 268         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 269         if (request == NULL)
 270                 return;
 271
 272         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 273         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 274         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 275         iscsi_pdu_queue_locked(request);
 276 }
 277
 278 static void
 279 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 280 {
 281         struct iscsi_outstanding *io, *tmp;
 282
 283         ISCSI_SESSION_LOCK_ASSERT(is);
 284
 285         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 286                 if (requeue) {
 287                         io->io_ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 288                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 289                 } else {
 290                         io->io_ccb->ccb_h.status = CAM_REQ_ABORTED;
 291                 }
 292
 293                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 294                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 295                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 296                 }
 297                 io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 298                 xpt_done(io->io_ccb);
 299                 iscsi_outstanding_remove(is, io);
 300         }
 301 }
 302
 303 static void
 304 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 305 {
 306         struct icl_pdu *pdu;
 307
 308         icl_conn_shutdown(is->is_conn);
 309         icl_conn_close(is->is_conn);
 310
 311         ISCSI_SESSION_LOCK(is);
 312
 313 #ifdef ICL_KERNEL_PROXY
 314         if (is->is_login_pdu != NULL) {
 315                 icl_pdu_free(is->is_login_pdu);
 316                 is->is_login_pdu = NULL;
 317         }
 318         cv_signal(&is->is_login_cv);
 319 #endif
 320
 321         /*
 322          * Don't queue any new PDUs.
 323          */
 324         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 325                 ISCSI_SESSION_DEBUG(is, "freezing");
 326                 xpt_freeze_simq(is->is_sim, 1);
 327                 is->is_simq_frozen = true;
 328         }
 329
 330         /*
 331          * Remove postponed PDUs.
 332          */
 333         while (!STAILQ_EMPTY(&is->is_postponed)) {
 334                 pdu = STAILQ_FIRST(&is->is_postponed);
 335                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 336                 icl_pdu_free(pdu);
 337         }
 338
 339         /*
 340          * Terminate SCSI tasks, asking CAM to requeue them.
 341          */
 342         //ISCSI_SESSION_DEBUG(is, "terminating tasks");
 343         iscsi_session_terminate_tasks(is, true);
 344
 345         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 346             ("destroying session with active tasks"));
 347         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 348             ("destroying session with postponed PDUs"));
 349
 350         /*
 351          * Request immediate reconnection from iscsid(8).
 352          */
 353         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 354         is->is_connected = false;
 355         is->is_reconnecting = false;
 356         is->is_login_phase = false;
 357         is->is_waiting_for_iscsid = true;
 358         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 359         is->is_timeout = 0;
 360         ISCSI_SESSION_UNLOCK(is);
 361         cv_signal(&is->is_softc->sc_cv);
 362 }
 363
 364 static void
 365 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 366 {
 367         struct iscsi_softc *sc;
 368         struct icl_pdu *pdu;
 369
 370         sc = is->is_softc;
 371         sx_xlock(&sc->sc_lock);
 372         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 373         sx_xunlock(&sc->sc_lock);
 374
 375         icl_conn_close(is->is_conn);
 376
 377         ISCSI_SESSION_LOCK(is);
 378
 379         KASSERT(is->is_terminating, ("is_terminating == false"));
 380
 381 #ifdef ICL_KERNEL_PROXY
 382         if (is->is_login_pdu != NULL) {
 383                 icl_pdu_free(is->is_login_pdu);
 384                 is->is_login_pdu = NULL;
 385         }
 386         cv_signal(&is->is_login_cv);
 387 #endif
 388
 389         /*
 390          * Don't queue any new PDUs.
 391          */
 392         callout_drain(&is->is_callout);
 393         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 394                 ISCSI_SESSION_DEBUG(is, "freezing");
 395                 xpt_freeze_simq(is->is_sim, 1);
 396                 is->is_simq_frozen = true;
 397         }
 398
 399         /*
 400          * Remove postponed PDUs.
 401          */
 402         while (!STAILQ_EMPTY(&is->is_postponed)) {
 403                 pdu = STAILQ_FIRST(&is->is_postponed);
 404                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
 405                 icl_pdu_free(pdu);
 406         }
 407
 408         /*
 409          * Forcibly terminate SCSI tasks.
 410          */
 411         ISCSI_SESSION_DEBUG(is, "terminating tasks");
 412         iscsi_session_terminate_tasks(is, false);
 413
 414         /*
 415          * Deregister CAM.
 416          */
 417         if (is->is_sim != NULL) {
 418                 ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 419                 xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 420
 421                 if (is->is_simq_frozen) {
 422                         xpt_release_simq(is->is_sim, 1);
 423                         is->is_simq_frozen = false;
 424                 }
 425
 426                 xpt_free_path(is->is_path);
 427                 xpt_bus_deregister(cam_sim_path(is->is_sim));
 428                 cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 429                 is->is_sim = NULL;
 430         }
 431
 432         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 433             ("destroying session with active tasks"));
 434         KASSERT(STAILQ_EMPTY(&is->is_postponed),
 435             ("destroying session with postponed PDUs"));
 436
 437         ISCSI_SESSION_UNLOCK(is);
 438
 439         icl_conn_free(is->is_conn);
 440         mtx_destroy(&is->is_lock);
 441         cv_destroy(&is->is_maintenance_cv);
 442 #ifdef ICL_KERNEL_PROXY
 443         cv_destroy(&is->is_login_cv);
 444 #endif
 445         ISCSI_SESSION_DEBUG(is, "terminated");
 446         free(is, M_ISCSI);
 447
 448         /*
 449          * The iscsi_unload() routine might be waiting.
 450          */
 451         cv_signal(&sc->sc_cv);
 452 }
 453
 454 static void
 455 iscsi_maintenance_thread(void *arg)
 456 {
 457         struct iscsi_session *is;
 458
 459         is = arg;
 460
 461         for (;;) {
 462                 ISCSI_SESSION_LOCK(is);
 463                 if (is->is_reconnecting == false &&
 464                     is->is_terminating == false &&
 465                     STAILQ_EMPTY(&is->is_postponed))
 466                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 467
 468                 if (is->is_reconnecting) {
 469                         ISCSI_SESSION_UNLOCK(is);
 470                         iscsi_maintenance_thread_reconnect(is);
 471                         continue;
 472                 }
 473
 474                 if (is->is_terminating) {
 475                         ISCSI_SESSION_UNLOCK(is);
 476                         iscsi_maintenance_thread_terminate(is);
 477                         kthread_exit();
 478                         return;
 479                 }
 480
 481                 iscsi_session_send_postponed(is);
 482                 ISCSI_SESSION_UNLOCK(is);
 483         }
 484 }
 485
 486 static void
 487 iscsi_session_reconnect(struct iscsi_session *is)
 488 {
 489
 490         /*
 491          * XXX: We can't use locking here, because
 492          *      it's being called from various contexts.
 493          *      Hope it doesn't break anything.
 494          */
 495         if (is->is_reconnecting)
 496                 return;
 497
 498         is->is_reconnecting = true;
 499         cv_signal(&is->is_maintenance_cv);
 500 }
 501
 502 static void
 503 iscsi_session_terminate(struct iscsi_session *is)
 504 {
 505         if (is->is_terminating)
 506                 return;
 507
 508         is->is_terminating = true;
 509
 510 #if 0
 511         iscsi_session_logout(is);
 512 #endif
 513         cv_signal(&is->is_maintenance_cv);
 514 }
 515
 516 static void
 517 iscsi_callout(void *context)
 518 {
 519         struct icl_pdu *request;
 520         struct iscsi_bhs_nop_out *bhsno;
 521         struct iscsi_session *is;
 522         bool reconnect_needed = false;
 523
 524         is = context;
 525
 526         if (is->is_terminating)
 527                 return;
 528
 529         callout_schedule(&is->is_callout, 1 * hz);
 530
 531         ISCSI_SESSION_LOCK(is);
 532         is->is_timeout++;
 533
 534         if (is->is_waiting_for_iscsid) {
 535                 if (is->is_timeout > iscsid_timeout) {
 536                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 537                             "for %d seconds; reconnecting",
 538                             is->is_timeout);
 539                         reconnect_needed = true;
 540                 }
 541                 goto out;
 542         }
 543
 544         if (is->is_login_phase) {
 545                 if (is->is_timeout > login_timeout) {
 546                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 547                             "reconnecting", is->is_timeout);
 548                         reconnect_needed = true;
 549                 }
 550                 goto out;
 551         }
 552
 553         if (is->is_timeout >= ping_timeout) {
 554                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 555                     "reconnecting", ping_timeout);
 556                 reconnect_needed = true;
 557                 goto out;
 558         }
 559
 560         ISCSI_SESSION_UNLOCK(is);
 561
 562         /*
 563          * If the ping was reset less than one second ago - which means
 564          * that we've received some PDU during the last second - assume
 565          * the traffic flows correctly and don't bother sending a NOP-Out.
 566          *
 567          * (It's 2 - one for one second, and one for incrementing is_timeout
 568          * earlier in this routine.)
 569          */
 570         if (is->is_timeout < 2)
 571                 return;
 572
 573         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 574         if (request == NULL) {
 575                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 576                 return;
 577         }
 578         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 579         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 580             ISCSI_BHS_OPCODE_IMMEDIATE;
 581         bhsno->bhsno_flags = 0x80;
 582         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 583         iscsi_pdu_queue(request);
 584         return;
 585
 586 out:
 587         ISCSI_SESSION_UNLOCK(is);
 588
 589         if (reconnect_needed)
 590                 iscsi_session_reconnect(is);
 591 }
 592
 593 static void
 594 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 595 {
 596         const struct iscsi_bhs_data_in *bhsdi;
 597         struct iscsi_session *is;
 598         uint32_t expcmdsn, maxcmdsn;
 599
 600         is = PDU_SESSION(response);
 601
 602         ISCSI_SESSION_LOCK_ASSERT(is);
 603
 604         /*
 605          * We're only using fields common for all the response
 606          * (target -> initiator) PDUs.
 607          */
 608         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 609         /*
 610          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 611          * and Residual Count only have meaningful content if the S bit
 612          * is set to 1", so we also need to check the bit specific for
 613          * Data-In PDU.
 614          */
 615         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 616             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 617                 if (ntohl(bhsdi->bhsdi_statsn) < is->is_statsn) {
 618                         ISCSI_SESSION_WARN(is,
 619                             "PDU StatSN %d >= session StatSN %d, opcode 0x%x",
 620                             is->is_statsn, ntohl(bhsdi->bhsdi_statsn),
 621                             bhsdi->bhsdi_opcode);
 622                 }
 623                 is->is_statsn = ntohl(bhsdi->bhsdi_statsn);
 624         }
 625
 626         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 627         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 628
 629         /*
 630          * XXX: Compare using Serial Arithmetic Sense.
 631          */
 632         if (maxcmdsn + 1 < expcmdsn) {
 633                 ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d + 1 < PDU ExpCmdSN %d; ignoring",
 634                     maxcmdsn, expcmdsn);
 635         } else {
 636                 if (maxcmdsn > is->is_maxcmdsn) {
 637                         is->is_maxcmdsn = maxcmdsn;
 638
 639                         /*
 640                          * Command window increased; kick the maintanance thread
 641                          * to send out postponed commands.
 642                          */
 643                         if (!STAILQ_EMPTY(&is->is_postponed))
 644                                 cv_signal(&is->is_maintenance_cv);
 645                 } else if (maxcmdsn < is->is_maxcmdsn) {
 646                         ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d < session MaxCmdSN %d; ignoring",
 647                             maxcmdsn, is->is_maxcmdsn);
 648                 }
 649
 650                 if (expcmdsn > is->is_expcmdsn) {
 651                         is->is_expcmdsn = expcmdsn;
 652                 } else if (expcmdsn < is->is_expcmdsn) {
 653                         ISCSI_SESSION_DEBUG(is, "PDU ExpCmdSN %d < session ExpCmdSN %d; ignoring",
 654                             expcmdsn, is->is_expcmdsn);
 655                 }
 656         }
 657
 658         /*
 659          * Every incoming PDU - not just NOP-In - resets the ping timer.
 660          * The purpose of the timeout is to reset the connection when it stalls;
 661          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 662          * in some queue.
 663          */
 664         is->is_timeout = 0;
 665 }
 666
 667 static void
 668 iscsi_receive_callback(struct icl_pdu *response)
 669 {
 670         struct iscsi_session *is;
 671
 672         is = PDU_SESSION(response);
 673
 674         ISCSI_SESSION_LOCK(is);
 675
 676 #ifdef ICL_KERNEL_PROXY
 677         if (is->is_login_phase) {
 678                 if (is->is_login_pdu == NULL)
 679                         is->is_login_pdu = response;
 680                 else
 681                         icl_pdu_free(response);
 682                 ISCSI_SESSION_UNLOCK(is);
 683                 cv_signal(&is->is_login_cv);
 684                 return;
 685         }
 686 #endif
 687
 688         iscsi_pdu_update_statsn(response);
 689
 690         /*
 691          * The handling routine is responsible for freeing the PDU
 692          * when it's no longer needed.
 693          */
 694         switch (response->ip_bhs->bhs_opcode) {
 695         case ISCSI_BHS_OPCODE_NOP_IN:
 696                 iscsi_pdu_handle_nop_in(response);
 697                 break;
 698         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 699                 iscsi_pdu_handle_scsi_response(response);
 700                 break;
 701         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 702                 iscsi_pdu_handle_data_in(response);
 703                 break;
 704         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 705                 iscsi_pdu_handle_logout_response(response);
 706                 break;
 707         case ISCSI_BHS_OPCODE_R2T:
 708                 iscsi_pdu_handle_r2t(response);
 709                 break;
 710         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 711                 iscsi_pdu_handle_async_message(response);
 712                 break;
 713         case ISCSI_BHS_OPCODE_REJECT:
 714                 iscsi_pdu_handle_reject(response);
 715                 break;
 716         default:
 717                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 718                     "opcode 0x%x; reconnecting",
 719                     response->ip_bhs->bhs_opcode);
 720                 iscsi_session_reconnect(is);
 721                 icl_pdu_free(response);
 722         }
 723
 724         ISCSI_SESSION_UNLOCK(is);
 725 }
 726
 727 static void
 728 iscsi_error_callback(struct icl_conn *ic)
 729 {
 730         struct iscsi_session *is;
 731
 732         is = CONN_SESSION(ic);
 733
 734         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 735         iscsi_session_reconnect(is);
 736 }
 737
 738 static void
 739 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 740 {
 741         struct iscsi_session *is;
 742         struct iscsi_bhs_nop_out *bhsno;
 743         struct iscsi_bhs_nop_in *bhsni;
 744         struct icl_pdu *request;
 745         void *data = NULL;
 746         size_t datasize;
 747         int error;
 748
 749         is = PDU_SESSION(response);
 750         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 751
 752         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 753                 /*
 754                  * Nothing to do; iscsi_pdu_update_statsn() already
 755                  * zeroed the timeout.
 756                  */
 757                 icl_pdu_free(response);
 758                 return;
 759         }
 760
 761         datasize = icl_pdu_data_segment_length(response);
 762         if (datasize > 0) {
 763                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 764                 if (data == NULL) {
 765                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 766                             "reconnecting");
 767                         icl_pdu_free(response);
 768                         iscsi_session_reconnect(is);
 769                         return;
 770                 }
 771                 icl_pdu_get_data(response, 0, data, datasize);
 772         }
 773
 774         request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
 775         if (request == NULL) {
 776                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 777                     "reconnecting");
 778                 free(data, M_ISCSI);
 779                 icl_pdu_free(response);
 780                 iscsi_session_reconnect(is);
 781                 return;
 782         }
 783         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 784         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 785             ISCSI_BHS_OPCODE_IMMEDIATE;
 786         bhsno->bhsno_flags = 0x80;
 787         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 788         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 789         if (datasize > 0) {
 790                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 791                 if (error != 0) {
 792                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 793                             "reconnecting");
 794                         free(data, M_ISCSI);
 795                         icl_pdu_free(request);
 796                         icl_pdu_free(response);
 797                         iscsi_session_reconnect(is);
 798                         return;
 799                 }
 800                 free(data, M_ISCSI);
 801         }
 802
 803         icl_pdu_free(response);
 804         iscsi_pdu_queue_locked(request);
 805 }
 806
 807 static void
 808 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 809 {
 810         struct iscsi_bhs_scsi_response *bhssr;
 811         struct iscsi_outstanding *io;
 812         struct iscsi_session *is;
 813         struct ccb_scsiio *csio;
 814         size_t data_segment_len;
 815         uint16_t sense_len;
 816
 817         is = PDU_SESSION(response);
 818
 819         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 820         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 821         if (io == NULL) {
 822                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 823                 icl_pdu_free(response);
 824                 iscsi_session_reconnect(is);
 825                 return;
 826         }
 827
 828         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 829                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 830                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 831                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 832                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 833                 }
 834                 io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 835         } else if (bhssr->bhssr_status == 0) {
 836                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
 837         } else {
 838                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 839                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 840                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 841                 }
 842                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 843                 io->io_ccb->csio.scsi_status = bhssr->bhssr_status;
 844         }
 845
 846         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
 847                 ISCSI_SESSION_WARN(is, "target indicated residual overflow");
 848                 icl_pdu_free(response);
 849                 iscsi_session_reconnect(is);
 850                 return;
 851         }
 852
 853         csio = &io->io_ccb->csio;
 854
 855         data_segment_len = icl_pdu_data_segment_length(response);
 856         if (data_segment_len > 0) {
 857                 if (data_segment_len < sizeof(sense_len)) {
 858                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 859                             data_segment_len);
 860                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 861                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 862                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 863                         }
 864                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 865                         goto out;
 866                 }
 867                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 868                 sense_len = ntohs(sense_len);
 869 #if 0
 870                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 871                     sense_len, data_segment_len);
 872 #endif
 873                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 874                         ISCSI_SESSION_WARN(is, "truncated data segment "
 875                             "(%zd bytes, should be %zd)",
 876                             data_segment_len, sizeof(sense_len) + sense_len);
 877                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 878                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 879                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 880                         }
 881                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 882                         goto out;
 883                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 884                         ISCSI_SESSION_WARN(is, "oversize data segment "
 885                             "(%zd bytes, should be %zd)",
 886                             data_segment_len, sizeof(sense_len) + sense_len);
 887                 if (sense_len > csio->sense_len) {
 888                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 889                             sense_len, csio->sense_len);
 890                         sense_len = csio->sense_len;
 891                 }
 892                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 893                 csio->sense_resid = csio->sense_len - sense_len;
 894                 io->io_ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 895         }
 896
 897 out:
 898         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 899                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 900
 901         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 902                 KASSERT(io->io_received <= csio->dxfer_len,
 903                     ("io->io_received > csio->dxfer_len"));
 904                 if (io->io_received < csio->dxfer_len) {
 905                         if (csio->resid != csio->dxfer_len - io->io_received) {
 906                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 907                                     "target indicates %d, we calculated %zd",
 908                                     csio->resid,
 909                                     csio->dxfer_len - io->io_received);
 910                         }
 911                         csio->resid = csio->dxfer_len - io->io_received;
 912                 }
 913         }
 914
 915         xpt_done(io->io_ccb);
 916         iscsi_outstanding_remove(is, io);
 917         icl_pdu_free(response);
 918 }
 919
 920 static void
 921 iscsi_pdu_handle_data_in(struct icl_pdu *response)
 922 {
 923         struct iscsi_bhs_data_in *bhsdi;
 924         struct iscsi_outstanding *io;
 925         struct iscsi_session *is;
 926         struct ccb_scsiio *csio;
 927         size_t data_segment_len;
 928
 929         is = PDU_SESSION(response);
 930         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
 931         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
 932         if (io == NULL) {
 933                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
 934                 icl_pdu_free(response);
 935                 iscsi_session_reconnect(is);
 936                 return;
 937         }
 938
 939         data_segment_len = icl_pdu_data_segment_length(response);
 940         if (data_segment_len == 0) {
 941                 /*
 942                  * "The sending of 0 length data segments should be avoided,
 943                  * but initiators and targets MUST be able to properly receive
 944                  * 0 length data segments."
 945                  */
 946                 icl_pdu_free(response);
 947                 return;
 948         }
 949
 950         /*
 951          * We need to track this for security reasons - without it, malicious target
 952          * could respond to SCSI READ without sending Data-In PDUs, which would result
 953          * in read operation on the initiator side returning random kernel data.
 954          */
 955         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
 956                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
 957                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
 958                 icl_pdu_free(response);
 959                 iscsi_session_reconnect(is);
 960                 return;
 961         }
 962
 963         csio = &io->io_ccb->csio;
 964
 965         if (io->io_received + data_segment_len > csio->dxfer_len) {
 966                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
 967                     "at offset %zd, buffer is %d)",
 968                     data_segment_len, io->io_received, csio->dxfer_len);
 969                 icl_pdu_free(response);
 970                 iscsi_session_reconnect(is);
 971                 return;
 972         }
 973
 974         icl_pdu_get_data(response, 0, csio->data_ptr + io->io_received, data_segment_len);
 975         io->io_received += data_segment_len;
 976
 977         /*
 978          * XXX: Check DataSN.
 979          * XXX: Check F.
 980          */
 981         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
 982                 /*
 983                  * Nothing more to do.
 984                  */
 985                 icl_pdu_free(response);
 986                 return;
 987         }
 988
 989         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
 990         if (bhsdi->bhsdi_status == 0) {
 991                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
 992         } else {
 993                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 994                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 995                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 996                 }
 997                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 998                 csio->scsi_status = bhsdi->bhsdi_status;
 999         }
1000
1001         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1002                 KASSERT(io->io_received <= csio->dxfer_len,
1003                     ("io->io_received > csio->dxfer_len"));
1004                 if (io->io_received < csio->dxfer_len) {
1005                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1006                         if (csio->resid != csio->dxfer_len - io->io_received) {
1007                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1008                                     "target indicates %d, we calculated %zd",
1009                                     csio->resid,
1010                                     csio->dxfer_len - io->io_received);
1011                         }
1012                         csio->resid = csio->dxfer_len - io->io_received;
1013                 }
1014         }
1015
1016         xpt_done(io->io_ccb);
1017         iscsi_outstanding_remove(is, io);
1018         icl_pdu_free(response);
1019 }
1020
1021 static void
1022 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1023 {
1024
1025         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1026         icl_pdu_free(response);
1027 }
1028
1029 static void
1030 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1031 {
1032         struct icl_pdu *request;
1033         struct iscsi_session *is;
1034         struct iscsi_bhs_r2t *bhsr2t;
1035         struct iscsi_bhs_data_out *bhsdo;
1036         struct iscsi_outstanding *io;
1037         struct ccb_scsiio *csio;
1038         size_t off, len, total_len;
1039         int error;
1040
1041         is = PDU_SESSION(response);
1042
1043         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1044         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1045         if (io == NULL) {
1046                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1047                     bhsr2t->bhsr2t_initiator_task_tag);
1048                 icl_pdu_free(response);
1049                 iscsi_session_reconnect(is);
1050                 return;
1051         }
1052
1053         csio = &io->io_ccb->csio;
1054
1055         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1056                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1057                 icl_pdu_free(response);
1058                 iscsi_session_reconnect(is);
1059                 return;
1060         }
1061
1062         /*
1063          * XXX: Verify R2TSN.
1064          */
1065
1066         io->io_datasn = 0;
1067
1068         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1069         if (off > csio->dxfer_len) {
1070                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1071                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1072                 icl_pdu_free(response);
1073                 iscsi_session_reconnect(is);
1074                 return;
1075         }
1076
1077         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1078         if (total_len == 0 || total_len > csio->dxfer_len) {
1079                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1080                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1081                 icl_pdu_free(response);
1082                 iscsi_session_reconnect(is);
1083                 return;
1084         }
1085
1086         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1087
1088         for (;;) {
1089                 len = total_len;
1090
1091                 if (len > is->is_max_data_segment_length)
1092                         len = is->is_max_data_segment_length;
1093
1094                 if (off + len > csio->dxfer_len) {
1095                         ISCSI_SESSION_WARN(is, "target requested invalid "
1096                             "length/offset %zd, buffer is %d; reconnecting",
1097                             off + len, csio->dxfer_len);
1098                         icl_pdu_free(response);
1099                         iscsi_session_reconnect(is);
1100                         return;
1101                 }
1102
1103                 request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
1104                 if (request == NULL) {
1105                         icl_pdu_free(response);
1106                         iscsi_session_reconnect(is);
1107                         return;
1108                 }
1109
1110                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1111                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1112                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1113                 bhsdo->bhsdo_initiator_task_tag =
1114                     bhsr2t->bhsr2t_initiator_task_tag;
1115                 bhsdo->bhsdo_target_transfer_tag =
1116                     bhsr2t->bhsr2t_target_transfer_tag;
1117                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1118                 bhsdo->bhsdo_buffer_offset = htonl(off);
1119                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1120                     M_NOWAIT);
1121                 if (error != 0) {
1122                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1123                             "reconnecting");
1124                         icl_pdu_free(request);
1125                         icl_pdu_free(response);
1126                         iscsi_session_reconnect(is);
1127                         return;
1128                 }
1129
1130                 off += len;
1131                 total_len -= len;
1132
1133                 if (total_len == 0) {
1134                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1135                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1136                 } else {
1137                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1138                 }
1139
1140                 iscsi_pdu_queue_locked(request);
1141
1142                 if (total_len == 0)
1143                         break;
1144         }
1145
1146         icl_pdu_free(response);
1147 }
1148
1149 static void
1150 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1151 {
1152         struct iscsi_bhs_asynchronous_message *bhsam;
1153         struct iscsi_session *is;
1154
1155         is = PDU_SESSION(response);
1156         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1157         switch (bhsam->bhsam_async_event) {
1158         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1159                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1160                 iscsi_session_logout(is);
1161                 iscsi_session_terminate(is);
1162                 break;
1163         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1164                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1165                 break;
1166         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1167                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1168                 break;
1169         default:
1170                 /*
1171                  * XXX: Technically, we're obligated to also handle
1172                  *      parameter renegotiation.
1173                  */
1174                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1175                 break;
1176         }
1177
1178         icl_pdu_free(response);
1179 }
1180
1181 static void
1182 iscsi_pdu_handle_reject(struct icl_pdu *response)
1183 {
1184         struct iscsi_bhs_reject *bhsr;
1185         struct iscsi_session *is;
1186
1187         is = PDU_SESSION(response);
1188         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1189         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1190             bhsr->bhsr_reason);
1191
1192         icl_pdu_free(response);
1193 }
1194
1195 static int
1196 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1197     struct iscsi_daemon_request *request)
1198 {
1199         struct iscsi_session *is;
1200         int error;
1201
1202         sx_slock(&sc->sc_lock);
1203         for (;;) {
1204                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1205                         if (is->is_waiting_for_iscsid)
1206                                 break;
1207                 }
1208
1209                 if (is == NULL) {
1210                         /*
1211                          * No session requires attention from iscsid(8); wait.
1212                          */
1213                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1214                         if (error != 0) {
1215                                 sx_sunlock(&sc->sc_lock);
1216                                 return (error);
1217                         }
1218                         continue;
1219                 }
1220
1221                 ISCSI_SESSION_LOCK(is);
1222                 is->is_waiting_for_iscsid = false;
1223                 is->is_login_phase = true;
1224                 is->is_reason[0] = '\0';
1225                 ISCSI_SESSION_UNLOCK(is);
1226
1227                 request->idr_session_id = is->is_id;
1228                 memcpy(&request->idr_conf, &is->is_conf,
1229                     sizeof(request->idr_conf));
1230
1231                 sx_sunlock(&sc->sc_lock);
1232                 return (0);
1233         }
1234 }
1235
1236 static int
1237 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1238     struct iscsi_daemon_handoff *handoff)
1239 {
1240         struct iscsi_session *is;
1241         int error;
1242
1243         sx_slock(&sc->sc_lock);
1244
1245         /*
1246          * Find the session to hand off socket to.
1247          */
1248         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1249                 if (is->is_id == handoff->idh_session_id)
1250                         break;
1251         }
1252         if (is == NULL) {
1253                 sx_sunlock(&sc->sc_lock);
1254                 return (ESRCH);
1255         }
1256         ISCSI_SESSION_LOCK(is);
1257         if (is->is_conf.isc_discovery || is->is_terminating) {
1258                 ISCSI_SESSION_UNLOCK(is);
1259                 sx_sunlock(&sc->sc_lock);
1260                 return (EINVAL);
1261         }
1262         if (is->is_connected) {
1263                 /*
1264                  * This might have happened because another iscsid(8)
1265                  * instance handed off the connection in the meantime.
1266                  * Just return.
1267                  */
1268                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1269                     "session");
1270                 ISCSI_SESSION_UNLOCK(is);
1271                 sx_sunlock(&sc->sc_lock);
1272                 return (EBUSY);
1273         }
1274
1275         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1276             sizeof(is->is_target_alias));
1277         memcpy(is->is_isid, handoff->idh_isid, sizeof(is->is_isid));
1278         is->is_statsn = handoff->idh_statsn;
1279         is->is_initial_r2t = handoff->idh_initial_r2t;
1280         is->is_immediate_data = handoff->idh_immediate_data;
1281         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1282         is->is_max_burst_length = handoff->idh_max_burst_length;
1283         is->is_first_burst_length = handoff->idh_first_burst_length;
1284
1285         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1286                 is->is_conn->ic_header_crc32c = true;
1287         else
1288                 is->is_conn->ic_header_crc32c = false;
1289         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1290                 is->is_conn->ic_data_crc32c = true;
1291         else
1292                 is->is_conn->ic_data_crc32c = false;
1293
1294         is->is_cmdsn = 0;
1295         is->is_expcmdsn = 0;
1296         is->is_maxcmdsn = 0;
1297         is->is_waiting_for_iscsid = false;
1298         is->is_login_phase = false;
1299         is->is_timeout = 0;
1300         is->is_connected = true;
1301         is->is_reason[0] = '\0';
1302
1303         ISCSI_SESSION_UNLOCK(is);
1304
1305 #ifndef ICL_KERNEL_PROXY
1306         error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1307         if (error != 0) {
1308                 sx_sunlock(&sc->sc_lock);
1309                 iscsi_session_terminate(is);
1310                 return (error);
1311         }
1312 #endif
1313
1314         sx_sunlock(&sc->sc_lock);
1315
1316         if (is->is_sim != NULL) {
1317                 /*
1318                  * When reconnecting, there already is SIM allocated for the session.
1319                  */
1320                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1321                 ISCSI_SESSION_LOCK(is);
1322                 ISCSI_SESSION_DEBUG(is, "releasing");
1323                 xpt_release_simq(is->is_sim, 1);
1324                 is->is_simq_frozen = false;
1325                 ISCSI_SESSION_UNLOCK(is);
1326
1327         } else {
1328                 ISCSI_SESSION_LOCK(is);
1329                 is->is_devq = cam_simq_alloc(maxtags);
1330                 if (is->is_devq == NULL) {
1331                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1332                         iscsi_session_terminate(is);
1333                         return (ENOMEM);
1334                 }
1335
1336                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1337                     is, is->is_id /* unit */, &is->is_lock,
1338                     maxtags, maxtags, is->is_devq);
1339                 if (is->is_sim == NULL) {
1340                         ISCSI_SESSION_UNLOCK(is);
1341                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1342                         cam_simq_free(is->is_devq);
1343                         iscsi_session_terminate(is);
1344                         return (ENOMEM);
1345                 }
1346
1347                 error = xpt_bus_register(is->is_sim, NULL, 0);
1348                 if (error != 0) {
1349                         ISCSI_SESSION_UNLOCK(is);
1350                         ISCSI_SESSION_WARN(is, "failed to register bus");
1351                         iscsi_session_terminate(is);
1352                         return (ENOMEM);
1353                 }
1354
1355                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1356                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1357                     CAM_LUN_WILDCARD);
1358                 if (error != CAM_REQ_CMP) {
1359                         ISCSI_SESSION_UNLOCK(is);
1360                         ISCSI_SESSION_WARN(is, "failed to create path");
1361                         iscsi_session_terminate(is);
1362                         return (ENOMEM);
1363                 }
1364                 ISCSI_SESSION_UNLOCK(is);
1365         }
1366
1367         return (0);
1368 }
1369
1370 static int
1371 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1372     struct iscsi_daemon_fail *fail)
1373 {
1374         struct iscsi_session *is;
1375
1376         sx_slock(&sc->sc_lock);
1377
1378         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1379                 if (is->is_id == fail->idf_session_id)
1380                         break;
1381         }
1382         if (is == NULL) {
1383                 sx_sunlock(&sc->sc_lock);
1384                 return (ESRCH);
1385         }
1386         ISCSI_SESSION_LOCK(is);
1387         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1388             fail->idf_reason);
1389         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1390         //is->is_waiting_for_iscsid = false;
1391         //is->is_login_phase = true;
1392         //iscsi_session_reconnect(is);
1393         ISCSI_SESSION_UNLOCK(is);
1394         sx_sunlock(&sc->sc_lock);
1395
1396         return (0);
1397 }
1398
1399 #ifdef ICL_KERNEL_PROXY
1400 static int
1401 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1402     struct iscsi_daemon_connect *idc)
1403 {
1404         struct iscsi_session *is;
1405         struct sockaddr *from_sa, *to_sa;
1406         int error;
1407
1408         sx_slock(&sc->sc_lock);
1409         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1410                 if (is->is_id == idc->idc_session_id)
1411                         break;
1412         }
1413         if (is == NULL) {
1414                 sx_sunlock(&sc->sc_lock);
1415                 return (ESRCH);
1416         }
1417         sx_sunlock(&sc->sc_lock);
1418
1419         if (idc->idc_from_addrlen > 0) {
1420                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1421                 if (error != 0)
1422                         return (error);
1423         } else {
1424                 from_sa = NULL;
1425         }
1426         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1427         if (error != 0) {
1428                 free(from_sa, M_SONAME);
1429                 return (error);
1430         }
1431
1432         ISCSI_SESSION_LOCK(is);
1433         is->is_waiting_for_iscsid = false;
1434         is->is_login_phase = true;
1435         is->is_timeout = 0;
1436         ISCSI_SESSION_UNLOCK(is);
1437
1438         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1439             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1440         free(from_sa, M_SONAME);
1441         free(to_sa, M_SONAME);
1442
1443         /*
1444          * Digests are always disabled during login phase.
1445          */
1446         is->is_conn->ic_header_crc32c = false;
1447         is->is_conn->ic_data_crc32c = false;
1448
1449         return (error);
1450 }
1451
1452 static int
1453 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1454     struct iscsi_daemon_send *ids)
1455 {
1456         struct iscsi_session *is;
1457         struct icl_pdu *ip;
1458         size_t datalen;
1459         void *data;
1460         int error;
1461
1462         sx_slock(&sc->sc_lock);
1463         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1464                 if (is->is_id == ids->ids_session_id)
1465                         break;
1466         }
1467         if (is == NULL) {
1468                 sx_sunlock(&sc->sc_lock);
1469                 return (ESRCH);
1470         }
1471         sx_sunlock(&sc->sc_lock);
1472
1473         if (is->is_login_phase == false)
1474                 return (EBUSY);
1475
1476         if (is->is_terminating || is->is_reconnecting)
1477                 return (EIO);
1478
1479         datalen = ids->ids_data_segment_len;
1480         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1481                 return (EINVAL);
1482         if (datalen > 0) {
1483                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1484                 error = copyin(ids->ids_data_segment, data, datalen);
1485                 if (error != 0) {
1486                         free(data, M_ISCSI);
1487                         return (error);
1488                 }
1489         }
1490
1491         ip = icl_pdu_new_bhs(is->is_conn, M_WAITOK);
1492         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1493         if (datalen > 0) {
1494                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1495                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1496                 free(data, M_ISCSI);
1497         }
1498         icl_pdu_queue(ip);
1499
1500         return (0);
1501 }
1502
1503 static int
1504 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1505     struct iscsi_daemon_receive *idr)
1506 {
1507         struct iscsi_session *is;
1508         struct icl_pdu *ip;
1509         void *data;
1510
1511         sx_slock(&sc->sc_lock);
1512         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1513                 if (is->is_id == idr->idr_session_id)
1514                         break;
1515         }
1516         if (is == NULL) {
1517                 sx_sunlock(&sc->sc_lock);
1518                 return (ESRCH);
1519         }
1520         sx_sunlock(&sc->sc_lock);
1521
1522         if (is->is_login_phase == false)
1523                 return (EBUSY);
1524
1525         ISCSI_SESSION_LOCK(is);
1526         while (is->is_login_pdu == NULL &&
1527             is->is_terminating == false &&
1528             is->is_reconnecting == false)
1529                 cv_wait(&is->is_login_cv, &is->is_lock);
1530         if (is->is_terminating || is->is_reconnecting) {
1531                 ISCSI_SESSION_UNLOCK(is);
1532                 return (EIO);
1533         }
1534         ip = is->is_login_pdu;
1535         is->is_login_pdu = NULL;
1536         ISCSI_SESSION_UNLOCK(is);
1537
1538         if (ip->ip_data_len > idr->idr_data_segment_len) {
1539                 icl_pdu_free(ip);
1540                 return (EMSGSIZE);
1541         }
1542
1543         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1544         if (ip->ip_data_len > 0) {
1545                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1546                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1547                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1548                 free(data, M_ISCSI);
1549         }
1550
1551         icl_pdu_free(ip);
1552
1553         return (0);
1554 }
1555 #endif /* ICL_KERNEL_PROXY */
1556
1557 static void
1558 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1559 {
1560         /*
1561          * Just make sure all the fields are null-terminated.
1562          *
1563          * XXX: This is not particularly secure.  We should
1564          *      create our own conf and then copy in relevant
1565          *      fields.
1566          */
1567         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1568         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1569         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1570         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1571         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1572         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1573         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1574         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1575         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1576 }
1577
1578 static int
1579 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1580 {
1581         struct iscsi_session *is;
1582         const struct iscsi_session *is2;
1583         int error;
1584
1585         iscsi_sanitize_session_conf(&isa->isa_conf);
1586
1587         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1588         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1589
1590         if (is->is_conf.isc_initiator[0] == '\0' ||
1591             is->is_conf.isc_target_addr[0] == '\0') {
1592                 free(is, M_ISCSI);
1593                 return (EINVAL);
1594         }
1595
1596         if ((is->is_conf.isc_discovery != 0 && is->is_conf.isc_target[0] != 0) ||
1597             (is->is_conf.isc_discovery == 0 && is->is_conf.isc_target[0] == 0)) {
1598                 free(is, M_ISCSI);
1599                 return (EINVAL);
1600         }
1601
1602         sx_xlock(&sc->sc_lock);
1603
1604         /*
1605          * Prevent duplicates.
1606          */
1607         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1608                 if (!!is->is_conf.isc_discovery !=
1609                     !!is2->is_conf.isc_discovery)
1610                         continue;
1611
1612                 if (strcmp(is->is_conf.isc_target_addr,
1613                     is2->is_conf.isc_target_addr) != 0)
1614                         continue;
1615
1616                 if (is->is_conf.isc_discovery == 0 &&
1617                     strcmp(is->is_conf.isc_target,
1618                     is2->is_conf.isc_target) != 0)
1619                         continue;
1620
1621                 sx_xunlock(&sc->sc_lock);
1622                 free(is, M_ISCSI);
1623                 return (EBUSY);
1624         }
1625
1626         is->is_conn = icl_conn_new("iscsi", &is->is_lock);
1627         is->is_conn->ic_receive = iscsi_receive_callback;
1628         is->is_conn->ic_error = iscsi_error_callback;
1629         is->is_conn->ic_prv0 = is;
1630         TAILQ_INIT(&is->is_outstanding);
1631         STAILQ_INIT(&is->is_postponed);
1632         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1633         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1634 #ifdef ICL_KERNEL_PROXY
1635         cv_init(&is->is_login_cv, "iscsi_login");
1636 #endif
1637
1638         is->is_softc = sc;
1639         sc->sc_last_session_id++;
1640         is->is_id = sc->sc_last_session_id;
1641         callout_init(&is->is_callout, 1);
1642         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1643         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1644
1645         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1646         if (error != 0) {
1647                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1648                 return (error);
1649         }
1650
1651         /*
1652          * Trigger immediate reconnection.
1653          */
1654         is->is_waiting_for_iscsid = true;
1655         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1656         cv_signal(&sc->sc_cv);
1657
1658         sx_xunlock(&sc->sc_lock);
1659
1660         return (0);
1661 }
1662
1663 static bool
1664 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1665     unsigned int id2, const struct iscsi_session_conf *c2)
1666 {
1667         if (id2 == 0 && c2->isc_target[0] == '\0' &&
1668             c2->isc_target_addr[0] == '\0')
1669                 return (true);
1670         if (id2 != 0 && id2 == id1)
1671                 return (true);
1672         if (c2->isc_target[0] != '\0' &&
1673             strcmp(c1->isc_target, c2->isc_target) == 0)
1674                 return (true);
1675         if (c2->isc_target_addr[0] != '\0' &&
1676             strcmp(c1->isc_target_addr, c2->isc_target_addr) == 0)
1677                 return (true);
1678         return (false);
1679 }
1680
1681 static int
1682 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1683     struct iscsi_session_remove *isr)
1684 {
1685         struct iscsi_session *is, *tmp;
1686         bool found = false;
1687
1688         iscsi_sanitize_session_conf(&isr->isr_conf);
1689
1690         sx_xlock(&sc->sc_lock);
1691         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1692                 ISCSI_SESSION_LOCK(is);
1693                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1694                     isr->isr_session_id, &isr->isr_conf)) {
1695                         found = true;
1696                         iscsi_session_logout(is);
1697                         iscsi_session_terminate(is);
1698                 }
1699                 ISCSI_SESSION_UNLOCK(is);
1700         }
1701         sx_xunlock(&sc->sc_lock);
1702
1703         if (!found)
1704                 return (ESRCH);
1705
1706         return (0);
1707 }
1708
1709 static int
1710 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1711 {
1712         int error;
1713         unsigned int i = 0;
1714         struct iscsi_session *is;
1715         struct iscsi_session_state iss;
1716
1717         sx_slock(&sc->sc_lock);
1718         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1719                 if (i >= isl->isl_nentries) {
1720                         sx_sunlock(&sc->sc_lock);
1721                         return (EMSGSIZE);
1722                 }
1723                 memset(&iss, 0, sizeof(iss));
1724                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1725                 iss.iss_id = is->is_id;
1726                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1727                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1728
1729                 if (is->is_conn->ic_header_crc32c)
1730                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1731                 else
1732                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1733
1734                 if (is->is_conn->ic_data_crc32c)
1735                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1736                 else
1737                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1738
1739                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1740                 iss.iss_immediate_data = is->is_immediate_data;
1741                 iss.iss_connected = is->is_connected;
1742
1743                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1744                 if (error != 0) {
1745                         sx_sunlock(&sc->sc_lock);
1746                         return (error);
1747                 }
1748                 i++;
1749         }
1750         sx_sunlock(&sc->sc_lock);
1751
1752         isl->isl_nentries = i;
1753
1754         return (0);
1755 }
1756
1757 static int
1758 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1759     struct thread *td)
1760 {
1761         struct iscsi_softc *sc;
1762
1763         sc = dev->si_drv1;
1764
1765         switch (cmd) {
1766         case ISCSIDWAIT:
1767                 return (iscsi_ioctl_daemon_wait(sc,
1768                     (struct iscsi_daemon_request *)arg));
1769         case ISCSIDHANDOFF:
1770                 return (iscsi_ioctl_daemon_handoff(sc,
1771                     (struct iscsi_daemon_handoff *)arg));
1772         case ISCSIDFAIL:
1773                 return (iscsi_ioctl_daemon_fail(sc,
1774                     (struct iscsi_daemon_fail *)arg));
1775 #ifdef ICL_KERNEL_PROXY
1776         case ISCSIDCONNECT:
1777                 return (iscsi_ioctl_daemon_connect(sc,
1778                     (struct iscsi_daemon_connect *)arg));
1779         case ISCSIDSEND:
1780                 return (iscsi_ioctl_daemon_send(sc,
1781                     (struct iscsi_daemon_send *)arg));
1782         case ISCSIDRECEIVE:
1783                 return (iscsi_ioctl_daemon_receive(sc,
1784                     (struct iscsi_daemon_receive *)arg));
1785 #endif /* ICL_KERNEL_PROXY */
1786         case ISCSISADD:
1787                 return (iscsi_ioctl_session_add(sc,
1788                     (struct iscsi_session_add *)arg));
1789         case ISCSISREMOVE:
1790                 return (iscsi_ioctl_session_remove(sc,
1791                     (struct iscsi_session_remove *)arg));
1792         case ISCSISLIST:
1793                 return (iscsi_ioctl_session_list(sc,
1794                     (struct iscsi_session_list *)arg));
1795         default:
1796                 return (EINVAL);
1797         }
1798 }
1799
1800 static struct iscsi_outstanding *
1801 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1802 {
1803         struct iscsi_outstanding *io;
1804
1805         ISCSI_SESSION_LOCK_ASSERT(is);
1806
1807         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1808                 if (io->io_initiator_task_tag == initiator_task_tag)
1809                         return (io);
1810         }
1811         return (NULL);
1812 }
1813
1814 static int
1815 iscsi_outstanding_add(struct iscsi_session *is,
1816     uint32_t initiator_task_tag, union ccb *ccb)
1817 {
1818         struct iscsi_outstanding *io;
1819
1820         ISCSI_SESSION_LOCK_ASSERT(is);
1821
1822         KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
1823             ("initiator_task_tag 0x%x already added", initiator_task_tag));
1824
1825         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
1826         if (io == NULL) {
1827                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
1828                 return (ENOMEM);
1829         }
1830         io->io_initiator_task_tag = initiator_task_tag;
1831         io->io_ccb = ccb;
1832         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
1833         return (0);
1834 }
1835
1836 static void
1837 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
1838 {
1839
1840         ISCSI_SESSION_LOCK_ASSERT(is);
1841
1842         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
1843         uma_zfree(iscsi_outstanding_zone, io);
1844 }
1845
1846 static void
1847 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
1848 {
1849         struct icl_pdu *request;
1850         struct iscsi_bhs_scsi_command *bhssc;
1851         struct ccb_scsiio *csio;
1852         size_t len;
1853         int error;
1854
1855         ISCSI_SESSION_LOCK_ASSERT(is);
1856
1857 #if 0
1858         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
1859 #else
1860         if (is->is_login_phase) {
1861                 ISCSI_SESSION_DEBUG(is, "called during login phase");
1862                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1863                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1864                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1865                 }
1866                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
1867                 xpt_done(ccb);
1868                 return;
1869         }
1870 #endif
1871
1872         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
1873         if (request == NULL) {
1874                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1875                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1876                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1877                 }
1878                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1879                 xpt_done(ccb);
1880                 return;
1881         }
1882
1883         csio = &ccb->csio;
1884         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
1885         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
1886         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
1887         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1888         case CAM_DIR_IN:
1889                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
1890                 break;
1891         case CAM_DIR_OUT:
1892                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
1893                 break;
1894         }
1895
1896         switch (csio->tag_action) {
1897         case MSG_HEAD_OF_Q_TAG:
1898                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
1899                 break;
1900                 break;
1901         case MSG_ORDERED_Q_TAG:
1902                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
1903                 break;
1904         case MSG_ACA_TASK:
1905                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
1906                 break;
1907         case CAM_TAG_ACTION_NONE:
1908         case MSG_SIMPLE_Q_TAG:
1909         default:
1910                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
1911                 break;
1912         }
1913
1914         bhssc->bhssc_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
1915         bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
1916         is->is_initiator_task_tag++;
1917         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
1918         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
1919             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
1920
1921         if (csio->ccb_h.flags & CAM_CDB_POINTER)
1922                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
1923         else
1924                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
1925
1926         error = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
1927         if (error != 0) {
1928                 icl_pdu_free(request);
1929                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1930                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1931                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1932                 }
1933                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1934                 xpt_done(ccb);
1935                 return;
1936         }
1937
1938         if (is->is_immediate_data &&
1939             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
1940                 len = csio->dxfer_len;
1941                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
1942                 if (len > is->is_first_burst_length) {
1943                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
1944                         len = is->is_first_burst_length;
1945                 }
1946
1947                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
1948                 if (error != 0) {
1949                         icl_pdu_free(request);
1950                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1951                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
1952                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
1953                         }
1954                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1955                         xpt_done(ccb);
1956                         return;
1957                 }
1958         }
1959         iscsi_pdu_queue_locked(request);
1960 }
1961
1962 static void
1963 iscsi_action(struct cam_sim *sim, union ccb *ccb)
1964 {
1965         struct iscsi_session *is;
1966
1967         is = cam_sim_softc(sim);
1968
1969         ISCSI_SESSION_LOCK_ASSERT(is);
1970
1971         if (is->is_terminating) {
1972                 ISCSI_SESSION_DEBUG(is, "called during termination");
1973                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
1974                 xpt_done(ccb);
1975                 return;
1976         }
1977
1978         switch (ccb->ccb_h.func_code) {
1979         case XPT_PATH_INQ:
1980         {
1981                 struct ccb_pathinq *cpi = &ccb->cpi;
1982
1983                 cpi->version_num = 1;
1984                 cpi->hba_inquiry = PI_TAG_ABLE;
1985                 cpi->target_sprt = 0;
1986                 cpi->hba_misc = PIM_EXTLUNS;
1987                 cpi->hba_eng_cnt = 0;
1988                 cpi->max_target = 0;
1989                 cpi->max_lun = 0;
1990                 cpi->initiator_id = ~0;
1991                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1992                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
1993                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1994                 cpi->unit_number = cam_sim_unit(sim);
1995                 cpi->bus_id = cam_sim_bus(sim);
1996                 cpi->base_transfer_speed = 150000; /* XXX */
1997                 cpi->transport = XPORT_ISCSI;
1998                 cpi->transport_version = 0;
1999                 cpi->protocol = PROTO_SCSI;
2000                 cpi->protocol_version = SCSI_REV_SPC3;
2001                 cpi->maxio = MAXPHYS;
2002                 cpi->ccb_h.status = CAM_REQ_CMP;
2003                 break;
2004         }
2005         case XPT_CALC_GEOMETRY:
2006                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2007                 ccb->ccb_h.status = CAM_REQ_CMP;
2008                 break;
2009 #if 0
2010         /*
2011          * XXX: What's the point?
2012          */
2013         case XPT_RESET_BUS:
2014         case XPT_ABORT:
2015         case XPT_TERM_IO:
2016                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2017                 ccb->ccb_h.status = CAM_REQ_CMP;
2018                 break;
2019 #endif
2020         case XPT_SCSI_IO:
2021                 iscsi_action_scsiio(is, ccb);
2022                 return;
2023         default:
2024 #if 0
2025                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2026 #endif
2027                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2028                 break;
2029         }
2030         xpt_done(ccb);
2031 }
2032
2033 static void
2034 iscsi_poll(struct cam_sim *sim)
2035 {
2036
2037         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2038 }
2039
2040 static void
2041 iscsi_shutdown(struct iscsi_softc *sc)
2042 {
2043         struct iscsi_session *is;
2044
2045         ISCSI_DEBUG("removing all sessions due to shutdown");
2046
2047         sx_slock(&sc->sc_lock);
2048         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2049                 iscsi_session_terminate(is);
2050         sx_sunlock(&sc->sc_lock);
2051 }
2052
2053 static int
2054 iscsi_load(void)
2055 {
2056         int error;
2057
2058         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2059         sx_init(&sc->sc_lock, "iscsi");
2060         TAILQ_INIT(&sc->sc_sessions);
2061         cv_init(&sc->sc_cv, "iscsi_cv");
2062
2063         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2064             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2065             UMA_ALIGN_PTR, 0);
2066
2067         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2068             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2069         if (error != 0) {
2070                 ISCSI_WARN("failed to create device node, error %d", error);
2071                 return (error);
2072         }
2073         sc->sc_cdev->si_drv1 = sc;
2074
2075         /*
2076          * Note that this needs to get run before dashutdown().  Otherwise,
2077          * when rebooting with iSCSI session with outstanding requests,
2078          * but disconnected, dashutdown() will hang on cam_periph_runccb().
2079          */
2080         sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2081             iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2082
2083         return (0);
2084 }
2085
2086 static int
2087 iscsi_unload(void)
2088 {
2089         struct iscsi_session *is, *tmp;
2090
2091         if (sc->sc_cdev != NULL) {
2092                 ISCSI_DEBUG("removing device node");
2093                 destroy_dev(sc->sc_cdev);
2094                 ISCSI_DEBUG("device node removed");
2095         }
2096
2097         if (sc->sc_shutdown_eh != NULL)
2098                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2099
2100         sx_slock(&sc->sc_lock);
2101         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2102                 iscsi_session_terminate(is);
2103         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2104                 ISCSI_DEBUG("waiting for sessions to terminate");
2105                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2106         }
2107         ISCSI_DEBUG("all sessions terminated");
2108         sx_sunlock(&sc->sc_lock);
2109
2110         uma_zdestroy(iscsi_outstanding_zone);
2111         sx_destroy(&sc->sc_lock);
2112         cv_destroy(&sc->sc_cv);
2113         free(sc, M_ISCSI);
2114         return (0);
2115 }
2116
2117 static int
2118 iscsi_quiesce(void)
2119 {
2120         sx_slock(&sc->sc_lock);
2121         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2122                 sx_sunlock(&sc->sc_lock);
2123                 return (EBUSY);
2124         }
2125         sx_sunlock(&sc->sc_lock);
2126         return (0);
2127 }
2128
2129 static int
2130 iscsi_modevent(module_t mod, int what, void *arg)
2131 {
2132         int error;
2133
2134         switch (what) {
2135         case MOD_LOAD:
2136                 error = iscsi_load();
2137                 break;
2138         case MOD_UNLOAD:
2139                 error = iscsi_unload();
2140                 break;
2141         case MOD_QUIESCE:
2142                 error = iscsi_quiesce();
2143                 break;
2144         default:
2145                 error = EINVAL;
2146                 break;
2147         }
2148         return (error);
2149 }
2150
2151 moduledata_t iscsi_data = {
2152         "iscsi",
2153         iscsi_modevent,
2154         0
2155 };
2156
2157 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2158 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2159 MODULE_DEPEND(iscsi, icl, 1, 1, 1);