sys/dev/iscsi/iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 #include <sys/param.h>
  33 #include <sys/condvar.h>
  34 #include <sys/conf.h>
  35 #include <sys/eventhandler.h>
  36 #include <sys/file.h>
  37 #include <sys/kernel.h>
  38 #include <sys/kthread.h>
  39 #include <sys/lock.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mutex.h>
  42 #include <sys/module.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/systm.h>
  45 #include <sys/sx.h>
  46 #include <vm/uma.h>
  47
  48 #include <cam/cam.h>
  49 #include <cam/cam_ccb.h>
  50 #include <cam/cam_xpt.h>
  51 #include <cam/cam_debug.h>
  52 #include <cam/cam_sim.h>
  53 #include <cam/cam_xpt_sim.h>
  54 #include <cam/cam_xpt_periph.h>
  55 #include <cam/cam_periph.h>
  56 #include <cam/scsi/scsi_all.h>
  57 #include <cam/scsi/scsi_message.h>
  58
  59 #include "iscsi_ioctl.h"
  60 #include "iscsi.h"
  61 #include "icl.h"
  62 #include "iscsi_proto.h"
  63
  64 #ifdef ICL_KERNEL_PROXY
  65 #include <sys/socketvar.h>
  66 #endif
  67
  68 /*
  69  * XXX: This is global so the iscsi_unload() can access it.
  70  *      Think about how to do this properly.
  71  */
  72 static struct iscsi_softc       *sc;
  73
  74 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD, 0, "iSCSI initiator");
  75 static int debug = 1;
  76 TUNABLE_INT("kern.iscsi.debug", &debug);
  77 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RW,
  78     &debug, 2, "Enable debug messages");
  79 static int ping_timeout = 5;
  80 TUNABLE_INT("kern.iscsi.ping_timeout", &ping_timeout);
  81 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RW, &ping_timeout,
  82     5, "Timeout for ping (NOP-Out) requests, in seconds");
  83 static int iscsid_timeout = 60;
  84 TUNABLE_INT("kern.iscsi.iscsid_timeout", &iscsid_timeout);
  85 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RW, &iscsid_timeout,
  86     60, "Time to wait for iscsid(8) to handle reconnection, in seconds");
  87 static int login_timeout = 60;
  88 TUNABLE_INT("kern.iscsi.login_timeout", &login_timeout);
  89 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RW, &login_timeout,
  90     60, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
  91 static int maxtags = 255;
  92 TUNABLE_INT("kern.iscsi.maxtags", &maxtags);
  93 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RW, &maxtags,
  94     255, "Max number of IO requests queued");
  95
  96 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
  97 static uma_zone_t iscsi_outstanding_zone;
  98
  99 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
 100 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
 101
 102 #define ISCSI_DEBUG(X, ...)                                     \
 103         if (debug > 1) {                                        \
 104                 printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 105         } while (0)
 106
 107 #define ISCSI_WARN(X, ...)                                      \
 108         if (debug > 0) {                                        \
 109                 printf("WARNING: %s: " X "\n",                  \
 110                     __func__, ## __VA_ARGS__);                  \
 111         } while (0)
 112
 113 #define ISCSI_SESSION_DEBUG(S, X, ...)                          \
 114         if (debug > 1) {                                        \
 115                 printf("%s: %s (%s): " X "\n",                  \
 116                     __func__, S->is_conf.isc_target_addr,       \
 117                     S->is_conf.isc_target, ## __VA_ARGS__);     \
 118         } while (0)
 119
 120 #define ISCSI_SESSION_WARN(S, X, ...)                           \
 121         if (debug > 0) {                                        \
 122                 printf("WARNING: %s (%s): " X "\n",             \
 123                     S->is_conf.isc_target_addr,                 \
 124                     S->is_conf.isc_target, ## __VA_ARGS__);     \
 125         } while (0)
 126
 127 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
 128 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
 129 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
 130
 131 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
 132                     int mode, struct thread *td);
 133
 134 static struct cdevsw iscsi_cdevsw = {
 135      .d_version = D_VERSION,
 136      .d_ioctl   = iscsi_ioctl,
 137      .d_name    = "iscsi",
 138 };
 139
 140 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
 141 static void     iscsi_pdu_queue(struct icl_pdu *request);
 142 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
 143 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
 144 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
 145 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
 146 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
 147 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
 148 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
 149 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
 150 static void     iscsi_session_reconnect(struct iscsi_session *is);
 151 static void     iscsi_session_terminate(struct iscsi_session *is);
 152 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
 153 static void     iscsi_poll(struct cam_sim *sim);
 154 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
 155                     uint32_t initiator_task_tag);
 156 static int      iscsi_outstanding_add(struct iscsi_session *is,
 157                     uint32_t initiator_task_tag, union ccb *ccb);
 158 static void     iscsi_outstanding_remove(struct iscsi_session *is,
 159                     struct iscsi_outstanding *io);
 160
 161 static bool
 162 iscsi_pdu_prepare(struct icl_pdu *request)
 163 {
 164         struct iscsi_session *is;
 165         struct iscsi_bhs_scsi_command *bhssc;
 166
 167         is = PDU_SESSION(request);
 168
 169         ISCSI_SESSION_LOCK_ASSERT(is);
 170
 171         /*
 172          * We're only using fields common for all the request
 173          * (initiator -> target) PDUs.
 174          */
 175         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 176
 177         /*
 178          * Data-Out PDU does not contain CmdSN.
 179          */
 180         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 181                 if (is->is_cmdsn > is->is_maxcmdsn &&
 182                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 183                         /*
 184                          * Current MaxCmdSN prevents us from sending any more
 185                          * SCSI Command PDUs to the target; postpone the PDU.
 186                          * It will get resent by either iscsi_pdu_queue(),
 187                          * or by maintenance thread.
 188                          */
 189 #if 0
 190                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %d, ExpCmdSN %d, MaxCmdSN %d, opcode 0x%x",
 191                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn, bhssc->bhssc_opcode);
 192 #endif
 193                         return (true);
 194                 }
 195                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
 196                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 197                         is->is_cmdsn++;
 198         }
 199         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
 200
 201         return (false);
 202 }
 203
 204 static void
 205 iscsi_session_send_postponed(struct iscsi_session *is)
 206 {
 207         struct icl_pdu *request;
 208         bool postpone;
 209
 210         ISCSI_SESSION_LOCK_ASSERT(is);
 211
 212         while (!TAILQ_EMPTY(&is->is_postponed)) {
 213                 request = TAILQ_FIRST(&is->is_postponed);
 214                 postpone = iscsi_pdu_prepare(request);
 215                 if (postpone)
 216                         break;
 217                 TAILQ_REMOVE(&is->is_postponed, request, ip_next);
 218                 icl_pdu_queue(request);
 219         }
 220 }
 221
 222 static void
 223 iscsi_pdu_queue_locked(struct icl_pdu *request)
 224 {
 225         struct iscsi_session *is;
 226         bool postpone;
 227
 228         is = PDU_SESSION(request);
 229         ISCSI_SESSION_LOCK_ASSERT(is);
 230         iscsi_session_send_postponed(is);
 231         postpone = iscsi_pdu_prepare(request);
 232         if (postpone) {
 233                 TAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
 234                 return;
 235         }
 236         icl_pdu_queue(request);
 237 }
 238
 239 static void
 240 iscsi_pdu_queue(struct icl_pdu *request)
 241 {
 242         struct iscsi_session *is;
 243
 244         is = PDU_SESSION(request);
 245         ISCSI_SESSION_LOCK(is);
 246         iscsi_pdu_queue_locked(request);
 247         ISCSI_SESSION_UNLOCK(is);
 248 }
 249
 250 static void
 251 iscsi_session_logout(struct iscsi_session *is)
 252 {
 253         struct icl_pdu *request;
 254         struct iscsi_bhs_logout_request *bhslr;
 255
 256         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 257         if (request == NULL)
 258                 return;
 259
 260         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 261         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
 262         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
 263         iscsi_pdu_queue_locked(request);
 264 }
 265
 266 static void
 267 iscsi_session_terminate_tasks(struct iscsi_session *is, bool requeue)
 268 {
 269         struct iscsi_outstanding *io, *tmp;
 270
 271         ISCSI_SESSION_LOCK_ASSERT(is);
 272
 273         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
 274                 if (requeue) {
 275                         io->io_ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 276                         io->io_ccb->ccb_h.status |= CAM_REQUEUE_REQ;
 277                 } else {
 278                         io->io_ccb->ccb_h.status = CAM_REQ_ABORTED;
 279                 }
 280
 281                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 282                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 283                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 284                 }
 285                 io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
 286                 xpt_done(io->io_ccb);
 287                 iscsi_outstanding_remove(is, io);
 288         }
 289 }
 290
 291 static void
 292 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 293 {
 294         struct icl_pdu *pdu;
 295
 296         icl_conn_shutdown(is->is_conn);
 297         icl_conn_close(is->is_conn);
 298
 299         ISCSI_SESSION_LOCK(is);
 300
 301 #ifdef ICL_KERNEL_PROXY
 302         if (is->is_login_pdu != NULL) {
 303                 icl_pdu_free(is->is_login_pdu);
 304                 is->is_login_pdu = NULL;
 305         }
 306         cv_signal(&is->is_login_cv);
 307 #endif
 308
 309         /*
 310          * Don't queue any new PDUs.
 311          */
 312         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 313                 ISCSI_SESSION_DEBUG(is, "freezing");
 314                 xpt_freeze_simq(is->is_sim, 1);
 315                 is->is_simq_frozen = true;
 316         }
 317
 318         /*
 319          * Remove postponed PDUs.
 320          */
 321         while (!TAILQ_EMPTY(&is->is_postponed)) {
 322                 pdu = TAILQ_FIRST(&is->is_postponed);
 323                 TAILQ_REMOVE(&is->is_postponed, pdu, ip_next);
 324                 icl_pdu_free(pdu);
 325         }
 326
 327         /*
 328          * Terminate SCSI tasks, asking CAM to requeue them.
 329          */
 330         //ISCSI_SESSION_DEBUG(is, "terminating tasks");
 331         iscsi_session_terminate_tasks(is, true);
 332
 333         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 334             ("destroying session with active tasks"));
 335         KASSERT(TAILQ_EMPTY(&is->is_postponed),
 336             ("destroying session with postponed PDUs"));
 337
 338         /*
 339          * Request immediate reconnection from iscsid(8).
 340          */
 341         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
 342         is->is_connected = false;
 343         is->is_reconnecting = false;
 344         is->is_login_phase = false;
 345         is->is_waiting_for_iscsid = true;
 346         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
 347         is->is_timeout = 0;
 348         ISCSI_SESSION_UNLOCK(is);
 349         cv_signal(&is->is_softc->sc_cv);
 350 }
 351
 352 static void
 353 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
 354 {
 355         struct iscsi_softc *sc;
 356         struct icl_pdu *pdu;
 357
 358         sc = is->is_softc;
 359         sx_xlock(&sc->sc_lock);
 360         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
 361         sx_xunlock(&sc->sc_lock);
 362
 363         icl_conn_close(is->is_conn);
 364
 365         ISCSI_SESSION_LOCK(is);
 366
 367         KASSERT(is->is_terminating, ("is_terminating == false"));
 368
 369 #ifdef ICL_KERNEL_PROXY
 370         if (is->is_login_pdu != NULL) {
 371                 icl_pdu_free(is->is_login_pdu);
 372                 is->is_login_pdu = NULL;
 373         }
 374         cv_signal(&is->is_login_cv);
 375 #endif
 376
 377         /*
 378          * Don't queue any new PDUs.
 379          */
 380         callout_drain(&is->is_callout);
 381         if (is->is_sim != NULL && is->is_simq_frozen == false) {
 382                 ISCSI_SESSION_DEBUG(is, "freezing");
 383                 xpt_freeze_simq(is->is_sim, 1);
 384                 is->is_simq_frozen = true;
 385         }
 386
 387         /*
 388          * Remove postponed PDUs.
 389          */
 390         while (!TAILQ_EMPTY(&is->is_postponed)) {
 391                 pdu = TAILQ_FIRST(&is->is_postponed);
 392                 TAILQ_REMOVE(&is->is_postponed, pdu, ip_next);
 393                 icl_pdu_free(pdu);
 394         }
 395
 396         /*
 397          * Forcibly terminate SCSI tasks.
 398          */
 399         ISCSI_SESSION_DEBUG(is, "terminating tasks");
 400         iscsi_session_terminate_tasks(is, false);
 401
 402         /*
 403          * Deregister CAM.
 404          */
 405         if (is->is_sim != NULL) {
 406                 ISCSI_SESSION_DEBUG(is, "deregistering SIM");
 407                 xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
 408
 409                 if (is->is_simq_frozen) {
 410                         xpt_release_simq(is->is_sim, 1);
 411                         is->is_simq_frozen = false;
 412                 }
 413
 414                 xpt_free_path(is->is_path);
 415                 xpt_bus_deregister(cam_sim_path(is->is_sim));
 416                 cam_sim_free(is->is_sim, TRUE /*free_devq*/);
 417                 is->is_sim = NULL;
 418         }
 419
 420         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
 421             ("destroying session with active tasks"));
 422         KASSERT(TAILQ_EMPTY(&is->is_postponed),
 423             ("destroying session with postponed PDUs"));
 424
 425         ISCSI_SESSION_UNLOCK(is);
 426
 427         icl_conn_free(is->is_conn);
 428         mtx_destroy(&is->is_lock);
 429         cv_destroy(&is->is_maintenance_cv);
 430 #ifdef ICL_KERNEL_PROXY
 431         cv_destroy(&is->is_login_cv);
 432 #endif
 433         ISCSI_SESSION_DEBUG(is, "terminated");
 434         free(is, M_ISCSI);
 435
 436         /*
 437          * The iscsi_unload() routine might be waiting.
 438          */
 439         cv_signal(&sc->sc_cv);
 440 }
 441
 442 static void
 443 iscsi_maintenance_thread(void *arg)
 444 {
 445         struct iscsi_session *is;
 446
 447         is = arg;
 448
 449         for (;;) {
 450                 ISCSI_SESSION_LOCK(is);
 451                 if (is->is_reconnecting == false &&
 452                     is->is_terminating == false &&
 453                     TAILQ_EMPTY(&is->is_postponed))
 454                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
 455
 456                 if (is->is_reconnecting) {
 457                         ISCSI_SESSION_UNLOCK(is);
 458                         iscsi_maintenance_thread_reconnect(is);
 459                         continue;
 460                 }
 461
 462                 if (is->is_terminating) {
 463                         ISCSI_SESSION_UNLOCK(is);
 464                         iscsi_maintenance_thread_terminate(is);
 465                         kthread_exit();
 466                         return;
 467                 }
 468
 469                 iscsi_session_send_postponed(is);
 470                 ISCSI_SESSION_UNLOCK(is);
 471         }
 472 }
 473
 474 static void
 475 iscsi_session_reconnect(struct iscsi_session *is)
 476 {
 477
 478         /*
 479          * XXX: We can't use locking here, because
 480          *      it's being called from various contexts.
 481          *      Hope it doesn't break anything.
 482          */
 483         if (is->is_reconnecting)
 484                 return;
 485
 486         is->is_reconnecting = true;
 487         cv_signal(&is->is_maintenance_cv);
 488 }
 489
 490 static void
 491 iscsi_session_terminate(struct iscsi_session *is)
 492 {
 493         if (is->is_terminating)
 494                 return;
 495
 496         is->is_terminating = true;
 497
 498 #if 0
 499         iscsi_session_logout(is);
 500 #endif
 501         cv_signal(&is->is_maintenance_cv);
 502 }
 503
 504 static void
 505 iscsi_callout(void *context)
 506 {
 507         struct icl_pdu *request;
 508         struct iscsi_bhs_nop_out *bhsno;
 509         struct iscsi_session *is;
 510         bool reconnect_needed = false;
 511
 512         is = context;
 513
 514         if (is->is_terminating)
 515                 return;
 516
 517         callout_schedule(&is->is_callout, 1 * hz);
 518
 519         ISCSI_SESSION_LOCK(is);
 520         is->is_timeout++;
 521
 522         if (is->is_waiting_for_iscsid) {
 523                 if (is->is_timeout > iscsid_timeout) {
 524                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
 525                             "for %d seconds; reconnecting",
 526                             is->is_timeout);
 527                         reconnect_needed = true;
 528                 }
 529                 goto out;
 530         }
 531
 532         if (is->is_login_phase) {
 533                 if (is->is_timeout > login_timeout) {
 534                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
 535                             "reconnecting", is->is_timeout);
 536                         reconnect_needed = true;
 537                 }
 538                 goto out;
 539         }
 540
 541         if (is->is_timeout >= ping_timeout) {
 542                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
 543                     "reconnecting", ping_timeout);
 544                 reconnect_needed = true;
 545                 goto out;
 546         }
 547
 548         ISCSI_SESSION_UNLOCK(is);
 549
 550         /*
 551          * If the ping was reset less than one second ago - which means
 552          * that we've received some PDU during the last second - assume
 553          * the traffic flows correctly and don't bother sending a NOP-Out.
 554          *
 555          * (It's 2 - one for one second, and one for incrementing is_timeout
 556          * earlier in this routine.)
 557          */
 558         if (is->is_timeout < 2)
 559                 return;
 560
 561         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
 562         if (request == NULL) {
 563                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
 564                 return;
 565         }
 566         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 567         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 568             ISCSI_BHS_OPCODE_IMMEDIATE;
 569         bhsno->bhsno_flags = 0x80;
 570         bhsno->bhsno_target_transfer_tag = 0xffffffff;
 571         iscsi_pdu_queue(request);
 572         return;
 573
 574 out:
 575         ISCSI_SESSION_UNLOCK(is);
 576
 577         if (reconnect_needed)
 578                 iscsi_session_reconnect(is);
 579 }
 580
 581 static void
 582 iscsi_pdu_update_statsn(const struct icl_pdu *response)
 583 {
 584         const struct iscsi_bhs_data_in *bhsdi;
 585         struct iscsi_session *is;
 586         uint32_t expcmdsn, maxcmdsn;
 587
 588         is = PDU_SESSION(response);
 589
 590         ISCSI_SESSION_LOCK_ASSERT(is);
 591
 592         /*
 593          * We're only using fields common for all the response
 594          * (target -> initiator) PDUs.
 595          */
 596         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
 597         /*
 598          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
 599          * and Residual Count only have meaningful content if the S bit
 600          * is set to 1", so we also need to check the bit specific for
 601          * Data-In PDU.
 602          */
 603         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 604             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
 605                 if (ntohl(bhsdi->bhsdi_statsn) < is->is_statsn) {
 606                         ISCSI_SESSION_WARN(is,
 607                             "PDU StatSN %d >= session StatSN %d, opcode 0x%x",
 608                             is->is_statsn, ntohl(bhsdi->bhsdi_statsn),
 609                             bhsdi->bhsdi_opcode);
 610                 }
 611                 is->is_statsn = ntohl(bhsdi->bhsdi_statsn);
 612         }
 613
 614         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
 615         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
 616
 617         /*
 618          * XXX: Compare using Serial Arithmetic Sense.
 619          */
 620         if (maxcmdsn + 1 < expcmdsn) {
 621                 ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d + 1 < PDU ExpCmdSN %d; ignoring",
 622                     maxcmdsn, expcmdsn);
 623         } else {
 624                 if (maxcmdsn > is->is_maxcmdsn) {
 625                         is->is_maxcmdsn = maxcmdsn;
 626
 627                         /*
 628                          * Command window increased; kick the maintanance thread
 629                          * to send out postponed commands.
 630                          */
 631                         if (!TAILQ_EMPTY(&is->is_postponed))
 632                                 cv_signal(&is->is_maintenance_cv);
 633                 } else if (maxcmdsn < is->is_maxcmdsn) {
 634                         ISCSI_SESSION_DEBUG(is, "PDU MaxCmdSN %d < session MaxCmdSN %d; ignoring",
 635                             maxcmdsn, is->is_maxcmdsn);
 636                 }
 637
 638                 if (expcmdsn > is->is_expcmdsn) {
 639                         is->is_expcmdsn = expcmdsn;
 640                 } else if (expcmdsn < is->is_expcmdsn) {
 641                         ISCSI_SESSION_DEBUG(is, "PDU ExpCmdSN %d < session ExpCmdSN %d; ignoring",
 642                             expcmdsn, is->is_expcmdsn);
 643                 }
 644         }
 645
 646         /*
 647          * Every incoming PDU - not just NOP-In - resets the ping timer.
 648          * The purpose of the timeout is to reset the connection when it stalls;
 649          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 650          * in some queue.
 651          */
 652         is->is_timeout = 0;
 653 }
 654
 655 static void
 656 iscsi_receive_callback(struct icl_pdu *response)
 657 {
 658         struct iscsi_session *is;
 659
 660         is = PDU_SESSION(response);
 661
 662         ISCSI_SESSION_LOCK(is);
 663
 664 #ifdef ICL_KERNEL_PROXY
 665         if (is->is_login_phase) {
 666                 if (is->is_login_pdu == NULL)
 667                         is->is_login_pdu = response;
 668                 else
 669                         icl_pdu_free(response);
 670                 ISCSI_SESSION_UNLOCK(is);
 671                 cv_signal(&is->is_login_cv);
 672                 return;
 673         }
 674 #endif
 675
 676         iscsi_pdu_update_statsn(response);
 677
 678         /*
 679          * The handling routine is responsible for freeing the PDU
 680          * when it's no longer needed.
 681          */
 682         switch (response->ip_bhs->bhs_opcode) {
 683         case ISCSI_BHS_OPCODE_NOP_IN:
 684                 iscsi_pdu_handle_nop_in(response);
 685                 break;
 686         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
 687                 iscsi_pdu_handle_scsi_response(response);
 688                 break;
 689         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
 690                 iscsi_pdu_handle_data_in(response);
 691                 break;
 692         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
 693                 iscsi_pdu_handle_logout_response(response);
 694                 break;
 695         case ISCSI_BHS_OPCODE_R2T:
 696                 iscsi_pdu_handle_r2t(response);
 697                 break;
 698         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
 699                 iscsi_pdu_handle_async_message(response);
 700                 break;
 701         case ISCSI_BHS_OPCODE_REJECT:
 702                 iscsi_pdu_handle_reject(response);
 703                 break;
 704         default:
 705                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
 706                     "opcode 0x%x; reconnecting",
 707                     response->ip_bhs->bhs_opcode);
 708                 iscsi_session_reconnect(is);
 709                 icl_pdu_free(response);
 710         }
 711
 712         ISCSI_SESSION_UNLOCK(is);
 713 }
 714
 715 static void
 716 iscsi_error_callback(struct icl_conn *ic)
 717 {
 718         struct iscsi_session *is;
 719
 720         is = CONN_SESSION(ic);
 721
 722         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
 723         iscsi_session_reconnect(is);
 724 }
 725
 726 static void
 727 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
 728 {
 729         struct iscsi_session *is;
 730         struct iscsi_bhs_nop_out *bhsno;
 731         struct iscsi_bhs_nop_in *bhsni;
 732         struct icl_pdu *request;
 733         void *data = NULL;
 734         size_t datasize;
 735         int error;
 736
 737         is = PDU_SESSION(response);
 738         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 739
 740         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
 741                 /*
 742                  * Nothing to do; iscsi_pdu_update_statsn() already
 743                  * zeroed the timeout.
 744                  */
 745                 icl_pdu_free(response);
 746                 return;
 747         }
 748
 749         datasize = icl_pdu_data_segment_length(response);
 750         if (datasize > 0) {
 751                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
 752                 if (data == NULL) {
 753                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 754                             "reconnecting");
 755                         icl_pdu_free(response);
 756                         iscsi_session_reconnect(is);
 757                         return;
 758                 }
 759                 icl_pdu_get_data(response, 0, data, datasize);
 760         }
 761
 762         request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
 763         if (request == NULL) {
 764                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 765                     "reconnecting");
 766                 free(data, M_ISCSI);
 767                 icl_pdu_free(response);
 768                 iscsi_session_reconnect(is);
 769                 return;
 770         }
 771         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 772         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
 773             ISCSI_BHS_OPCODE_IMMEDIATE;
 774         bhsno->bhsno_flags = 0x80;
 775         bhsno->bhsno_initiator_task_tag = 0xffffffff;
 776         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
 777         if (datasize > 0) {
 778                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
 779                 if (error != 0) {
 780                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
 781                             "reconnecting");
 782                         free(data, M_ISCSI);
 783                         icl_pdu_free(request);
 784                         icl_pdu_free(response);
 785                         iscsi_session_reconnect(is);
 786                         return;
 787                 }
 788                 free(data, M_ISCSI);
 789         }
 790
 791         icl_pdu_free(response);
 792         iscsi_pdu_queue_locked(request);
 793 }
 794
 795 static void
 796 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
 797 {
 798         struct iscsi_bhs_scsi_response *bhssr;
 799         struct iscsi_outstanding *io;
 800         struct iscsi_session *is;
 801         struct ccb_scsiio *csio;
 802         size_t data_segment_len;
 803         uint16_t sense_len;
 804
 805         is = PDU_SESSION(response);
 806
 807         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 808         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
 809         if (io == NULL) {
 810                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
 811                 icl_pdu_free(response);
 812                 iscsi_session_reconnect(is);
 813                 return;
 814         }
 815
 816         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
 817                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
 818                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 819                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 820                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 821                 }
 822                 io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 823         } else if (bhssr->bhssr_status == 0) {
 824                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
 825         } else {
 826                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 827                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 828                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 829                 }
 830                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 831                 io->io_ccb->csio.scsi_status = bhssr->bhssr_status;
 832         }
 833
 834         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
 835                 ISCSI_SESSION_WARN(is, "target indicated residual overflow");
 836                 icl_pdu_free(response);
 837                 iscsi_session_reconnect(is);
 838                 return;
 839         }
 840
 841         csio = &io->io_ccb->csio;
 842
 843         data_segment_len = icl_pdu_data_segment_length(response);
 844         if (data_segment_len > 0) {
 845                 if (data_segment_len < sizeof(sense_len)) {
 846                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
 847                             data_segment_len);
 848                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 849                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 850                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 851                         }
 852                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 853                         goto out;
 854                 }
 855                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
 856                 sense_len = ntohs(sense_len);
 857 #if 0
 858                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
 859                     sense_len, data_segment_len);
 860 #endif
 861                 if (sizeof(sense_len) + sense_len > data_segment_len) {
 862                         ISCSI_SESSION_WARN(is, "truncated data segment "
 863                             "(%zd bytes, should be %zd)",
 864                             data_segment_len, sizeof(sense_len) + sense_len);
 865                         if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 866                                 xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 867                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
 868                         }
 869                         io->io_ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
 870                         goto out;
 871                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
 872                         ISCSI_SESSION_WARN(is, "oversize data segment "
 873                             "(%zd bytes, should be %zd)",
 874                             data_segment_len, sizeof(sense_len) + sense_len);
 875                 if (sense_len > csio->sense_len) {
 876                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
 877                             sense_len, csio->sense_len);
 878                         sense_len = csio->sense_len;
 879                 }
 880                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
 881                 csio->sense_resid = csio->sense_len - sense_len;
 882                 io->io_ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 883         }
 884
 885 out:
 886         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
 887                 csio->resid = ntohl(bhssr->bhssr_residual_count);
 888
 889         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 890                 KASSERT(io->io_received <= csio->dxfer_len,
 891                     ("io->io_received > csio->dxfer_len"));
 892                 if (io->io_received < csio->dxfer_len) {
 893                         if (csio->resid != csio->dxfer_len - io->io_received) {
 894                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 895                                     "target indicates %d, we calculated %zd",
 896                                     csio->resid,
 897                                     csio->dxfer_len - io->io_received);
 898                         }
 899                         csio->resid = csio->dxfer_len - io->io_received;
 900                 }
 901         }
 902
 903         xpt_done(io->io_ccb);
 904         iscsi_outstanding_remove(is, io);
 905         icl_pdu_free(response);
 906 }
 907
 908 static void
 909 iscsi_pdu_handle_data_in(struct icl_pdu *response)
 910 {
 911         struct iscsi_bhs_data_in *bhsdi;
 912         struct iscsi_outstanding *io;
 913         struct iscsi_session *is;
 914         struct ccb_scsiio *csio;
 915         size_t data_segment_len;
 916
 917         is = PDU_SESSION(response);
 918         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
 919         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
 920         if (io == NULL) {
 921                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
 922                 icl_pdu_free(response);
 923                 iscsi_session_reconnect(is);
 924                 return;
 925         }
 926
 927         data_segment_len = icl_pdu_data_segment_length(response);
 928         if (data_segment_len == 0) {
 929                 /*
 930                  * "The sending of 0 length data segments should be avoided,
 931                  * but initiators and targets MUST be able to properly receive
 932                  * 0 length data segments."
 933                  */
 934                 icl_pdu_free(response);
 935                 return;
 936         }
 937
 938         /*
 939          * We need to track this for security reasons - without it, malicious target
 940          * could respond to SCSI READ without sending Data-In PDUs, which would result
 941          * in read operation on the initiator side returning random kernel data.
 942          */
 943         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
 944                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
 945                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
 946                 icl_pdu_free(response);
 947                 iscsi_session_reconnect(is);
 948                 return;
 949         }
 950
 951         csio = &io->io_ccb->csio;
 952
 953         if (io->io_received + data_segment_len > csio->dxfer_len) {
 954                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
 955                     "at offset %zd, buffer is %d)",
 956                     data_segment_len, io->io_received, csio->dxfer_len);
 957                 icl_pdu_free(response);
 958                 iscsi_session_reconnect(is);
 959                 return;
 960         }
 961
 962         icl_pdu_get_data(response, 0, csio->data_ptr + io->io_received, data_segment_len);
 963         io->io_received += data_segment_len;
 964
 965         /*
 966          * XXX: Check DataSN.
 967          * XXX: Check F.
 968          */
 969         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
 970                 /*
 971                  * Nothing more to do.
 972                  */
 973                 icl_pdu_free(response);
 974                 return;
 975         }
 976
 977         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
 978         if (bhsdi->bhsdi_status == 0) {
 979                 io->io_ccb->ccb_h.status = CAM_REQ_CMP;
 980         } else {
 981                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 982                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
 983                         ISCSI_SESSION_DEBUG(is, "freezing devq");
 984                 }
 985                 io->io_ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
 986                 csio->scsi_status = bhsdi->bhsdi_status;
 987         }
 988
 989         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 990                 KASSERT(io->io_received <= csio->dxfer_len,
 991                     ("io->io_received > csio->dxfer_len"));
 992                 if (io->io_received < csio->dxfer_len) {
 993                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
 994                         if (csio->resid != csio->dxfer_len - io->io_received) {
 995                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
 996                                     "target indicates %d, we calculated %zd",
 997                                     csio->resid,
 998                                     csio->dxfer_len - io->io_received);
 999                         }
1000                         csio->resid = csio->dxfer_len - io->io_received;
1001                 }
1002         }
1003
1004         xpt_done(io->io_ccb);
1005         iscsi_outstanding_remove(is, io);
1006         icl_pdu_free(response);
1007 }
1008
1009 static void
1010 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1011 {
1012
1013         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1014         icl_pdu_free(response);
1015 }
1016
1017 static void
1018 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1019 {
1020         struct icl_pdu *request;
1021         struct iscsi_session *is;
1022         struct iscsi_bhs_r2t *bhsr2t;
1023         struct iscsi_bhs_data_out *bhsdo;
1024         struct iscsi_outstanding *io;
1025         struct ccb_scsiio *csio;
1026         size_t off, len, total_len;
1027         int error;
1028
1029         is = PDU_SESSION(response);
1030
1031         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1032         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1033         if (io == NULL) {
1034                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1035                     bhsr2t->bhsr2t_initiator_task_tag);
1036                 icl_pdu_free(response);
1037                 iscsi_session_reconnect(is);
1038                 return;
1039         }
1040
1041         csio = &io->io_ccb->csio;
1042
1043         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1044                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1045                 icl_pdu_free(response);
1046                 iscsi_session_reconnect(is);
1047                 return;
1048         }
1049
1050         /*
1051          * XXX: Verify R2TSN.
1052          */
1053
1054         io->io_datasn = 0;
1055
1056         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1057         if (off > csio->dxfer_len) {
1058                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1059                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1060                 icl_pdu_free(response);
1061                 iscsi_session_reconnect(is);
1062                 return;
1063         }
1064
1065         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1066         if (total_len == 0 || total_len > csio->dxfer_len) {
1067                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1068                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1069                 icl_pdu_free(response);
1070                 iscsi_session_reconnect(is);
1071                 return;
1072         }
1073
1074         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1075
1076         for (;;) {
1077                 len = total_len;
1078
1079                 if (len > is->is_max_data_segment_length)
1080                         len = is->is_max_data_segment_length;
1081
1082                 if (off + len > csio->dxfer_len) {
1083                         ISCSI_SESSION_WARN(is, "target requested invalid "
1084                             "length/offset %zd, buffer is %d; reconnecting",
1085                             off + len, csio->dxfer_len);
1086                         icl_pdu_free(response);
1087                         iscsi_session_reconnect(is);
1088                         return;
1089                 }
1090
1091                 request = icl_pdu_new_bhs(response->ip_conn, M_NOWAIT);
1092                 if (request == NULL) {
1093                         icl_pdu_free(response);
1094                         iscsi_session_reconnect(is);
1095                         return;
1096                 }
1097
1098                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1099                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1100                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1101                 bhsdo->bhsdo_initiator_task_tag =
1102                     bhsr2t->bhsr2t_initiator_task_tag;
1103                 bhsdo->bhsdo_target_transfer_tag =
1104                     bhsr2t->bhsr2t_target_transfer_tag;
1105                 bhsdo->bhsdo_datasn = htonl(io->io_datasn++);
1106                 bhsdo->bhsdo_buffer_offset = htonl(off);
1107                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1108                     M_NOWAIT);
1109                 if (error != 0) {
1110                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1111                             "reconnecting");
1112                         icl_pdu_free(request);
1113                         icl_pdu_free(response);
1114                         iscsi_session_reconnect(is);
1115                         return;
1116                 }
1117
1118                 off += len;
1119                 total_len -= len;
1120
1121                 if (total_len == 0) {
1122                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1123                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1124                 } else {
1125                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1126                 }
1127
1128                 iscsi_pdu_queue_locked(request);
1129
1130                 if (total_len == 0)
1131                         break;
1132         }
1133
1134         icl_pdu_free(response);
1135 }
1136
1137 static void
1138 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1139 {
1140         struct iscsi_bhs_asynchronous_message *bhsam;
1141         struct iscsi_session *is;
1142
1143         is = PDU_SESSION(response);
1144         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1145         switch (bhsam->bhsam_async_event) {
1146         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1147                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1148                 iscsi_session_logout(is);
1149                 iscsi_session_terminate(is);
1150                 break;
1151         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1152                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the connection");
1153                 break;
1154         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1155                 ISCSI_SESSION_WARN(is, "target indicates it will drop drop the session");
1156                 break;
1157         default:
1158                 /*
1159                  * XXX: Technically, we're obligated to also handle
1160                  *      parameter renegotiation.
1161                  */
1162                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1163                 break;
1164         }
1165
1166         icl_pdu_free(response);
1167 }
1168
1169 static void
1170 iscsi_pdu_handle_reject(struct icl_pdu *response)
1171 {
1172         struct iscsi_bhs_reject *bhsr;
1173         struct iscsi_session *is;
1174
1175         is = PDU_SESSION(response);
1176         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1177         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1178             bhsr->bhsr_reason);
1179
1180         icl_pdu_free(response);
1181 }
1182
1183 static int
1184 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1185     struct iscsi_daemon_request *request)
1186 {
1187         struct iscsi_session *is;
1188         int error;
1189
1190         sx_slock(&sc->sc_lock);
1191         for (;;) {
1192                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1193                         if (is->is_waiting_for_iscsid)
1194                                 break;
1195                 }
1196
1197                 if (is == NULL) {
1198                         /*
1199                          * No session requires attention from iscsid(8); wait.
1200                          */
1201                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1202                         if (error != 0) {
1203                                 sx_sunlock(&sc->sc_lock);
1204                                 return (error);
1205                         }
1206                         continue;
1207                 }
1208
1209                 ISCSI_SESSION_LOCK(is);
1210                 is->is_waiting_for_iscsid = false;
1211                 is->is_login_phase = true;
1212                 is->is_reason[0] = '\0';
1213                 ISCSI_SESSION_UNLOCK(is);
1214
1215                 request->idr_session_id = is->is_id;
1216                 memcpy(&request->idr_conf, &is->is_conf,
1217                     sizeof(request->idr_conf));
1218
1219                 sx_sunlock(&sc->sc_lock);
1220                 return (0);
1221         }
1222 }
1223
1224 static int
1225 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1226     struct iscsi_daemon_handoff *handoff)
1227 {
1228         struct iscsi_session *is;
1229         int error;
1230
1231         sx_slock(&sc->sc_lock);
1232
1233         /*
1234          * Find the session to hand off socket to.
1235          */
1236         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1237                 if (is->is_id == handoff->idh_session_id)
1238                         break;
1239         }
1240         if (is == NULL) {
1241                 sx_sunlock(&sc->sc_lock);
1242                 return (ESRCH);
1243         }
1244         ISCSI_SESSION_LOCK(is);
1245         if (is->is_conf.isc_discovery || is->is_terminating) {
1246                 ISCSI_SESSION_UNLOCK(is);
1247                 sx_sunlock(&sc->sc_lock);
1248                 return (EINVAL);
1249         }
1250
1251         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1252             sizeof(is->is_target_alias));
1253         memcpy(is->is_isid, handoff->idh_isid, sizeof(is->is_isid));
1254         is->is_statsn = handoff->idh_statsn;
1255         is->is_initial_r2t = handoff->idh_initial_r2t;
1256         is->is_immediate_data = handoff->idh_immediate_data;
1257         is->is_max_data_segment_length = handoff->idh_max_data_segment_length;
1258         is->is_max_burst_length = handoff->idh_max_burst_length;
1259         is->is_first_burst_length = handoff->idh_first_burst_length;
1260
1261         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1262                 is->is_conn->ic_header_crc32c = true;
1263         else
1264                 is->is_conn->ic_header_crc32c = false;
1265         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1266                 is->is_conn->ic_data_crc32c = true;
1267         else
1268                 is->is_conn->ic_data_crc32c = false;
1269
1270         is->is_cmdsn = 0;
1271         is->is_expcmdsn = 1;
1272         is->is_maxcmdsn = 1;
1273         is->is_waiting_for_iscsid = false;
1274         is->is_login_phase = false;
1275         is->is_timeout = 0;
1276         is->is_connected = true;
1277         is->is_reason[0] = '\0';
1278
1279         ISCSI_SESSION_UNLOCK(is);
1280
1281 #ifndef ICL_KERNEL_PROXY
1282         error = icl_conn_handoff(is->is_conn, handoff->idh_socket);
1283         if (error != 0) {
1284                 sx_sunlock(&sc->sc_lock);
1285                 iscsi_session_terminate(is);
1286                 return (error);
1287         }
1288 #endif
1289
1290         sx_sunlock(&sc->sc_lock);
1291
1292         if (is->is_sim != NULL) {
1293                 /*
1294                  * When reconnecting, there already is SIM allocated for the session.
1295                  */
1296                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1297                 ISCSI_SESSION_LOCK(is);
1298                 ISCSI_SESSION_DEBUG(is, "releasing");
1299                 xpt_release_simq(is->is_sim, 1);
1300                 is->is_simq_frozen = false;
1301                 ISCSI_SESSION_UNLOCK(is);
1302
1303         } else {
1304                 ISCSI_SESSION_LOCK(is);
1305                 is->is_devq = cam_simq_alloc(maxtags);
1306                 if (is->is_devq == NULL) {
1307                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1308                         iscsi_session_terminate(is);
1309                         return (ENOMEM);
1310                 }
1311
1312                 is->is_sim = cam_sim_alloc(iscsi_action, iscsi_poll, "iscsi",
1313                     is, is->is_id /* unit */, &is->is_lock,
1314                     maxtags, maxtags, is->is_devq);
1315                 if (is->is_sim == NULL) {
1316                         ISCSI_SESSION_UNLOCK(is);
1317                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1318                         cam_simq_free(is->is_devq);
1319                         iscsi_session_terminate(is);
1320                         return (ENOMEM);
1321                 }
1322
1323                 error = xpt_bus_register(is->is_sim, NULL, 0);
1324                 if (error != 0) {
1325                         ISCSI_SESSION_UNLOCK(is);
1326                         ISCSI_SESSION_WARN(is, "failed to register bus");
1327                         iscsi_session_terminate(is);
1328                         return (ENOMEM);
1329                 }
1330
1331                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1332                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1333                     CAM_LUN_WILDCARD);
1334                 if (error != CAM_REQ_CMP) {
1335                         ISCSI_SESSION_UNLOCK(is);
1336                         ISCSI_SESSION_WARN(is, "failed to create path");
1337                         iscsi_session_terminate(is);
1338                         return (ENOMEM);
1339                 }
1340                 ISCSI_SESSION_UNLOCK(is);
1341         }
1342
1343         return (0);
1344 }
1345
1346 static int
1347 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1348     struct iscsi_daemon_fail *fail)
1349 {
1350         struct iscsi_session *is;
1351
1352         sx_slock(&sc->sc_lock);
1353
1354         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1355                 if (is->is_id == fail->idf_session_id)
1356                         break;
1357         }
1358         if (is == NULL) {
1359                 sx_sunlock(&sc->sc_lock);
1360                 return (ESRCH);
1361         }
1362         ISCSI_SESSION_LOCK(is);
1363         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1364             fail->idf_reason);
1365         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1366         //is->is_waiting_for_iscsid = false;
1367         //is->is_login_phase = true;
1368         //iscsi_session_reconnect(is);
1369         ISCSI_SESSION_UNLOCK(is);
1370         sx_sunlock(&sc->sc_lock);
1371
1372         return (0);
1373 }
1374
1375 #ifdef ICL_KERNEL_PROXY
1376 static int
1377 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1378     struct iscsi_daemon_connect *idc)
1379 {
1380         struct iscsi_session *is;
1381         struct sockaddr *from_sa, *to_sa;
1382         int error;
1383
1384         sx_slock(&sc->sc_lock);
1385         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1386                 if (is->is_id == idc->idc_session_id)
1387                         break;
1388         }
1389         if (is == NULL) {
1390                 sx_sunlock(&sc->sc_lock);
1391                 return (ESRCH);
1392         }
1393         sx_sunlock(&sc->sc_lock);
1394
1395         if (idc->idc_from_addrlen > 0) {
1396                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1397                 if (error != 0)
1398                         return (error);
1399         } else {
1400                 from_sa = NULL;
1401         }
1402         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1403         if (error != 0) {
1404                 free(from_sa, M_SONAME);
1405                 return (error);
1406         }
1407
1408         ISCSI_SESSION_LOCK(is);
1409         is->is_waiting_for_iscsid = false;
1410         is->is_login_phase = true;
1411         is->is_timeout = 0;
1412         ISCSI_SESSION_UNLOCK(is);
1413
1414         error = icl_conn_connect(is->is_conn, idc->idc_iser, idc->idc_domain,
1415             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1416         free(from_sa, M_SONAME);
1417         free(to_sa, M_SONAME);
1418
1419         /*
1420          * Digests are always disabled during login phase.
1421          */
1422         is->is_conn->ic_header_crc32c = false;
1423         is->is_conn->ic_data_crc32c = false;
1424
1425         return (error);
1426 }
1427
1428 static int
1429 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1430     struct iscsi_daemon_send *ids)
1431 {
1432         struct iscsi_session *is;
1433         struct icl_pdu *ip;
1434         size_t datalen;
1435         void *data;
1436         int error;
1437
1438         sx_slock(&sc->sc_lock);
1439         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1440                 if (is->is_id == ids->ids_session_id)
1441                         break;
1442         }
1443         if (is == NULL) {
1444                 sx_sunlock(&sc->sc_lock);
1445                 return (ESRCH);
1446         }
1447         sx_sunlock(&sc->sc_lock);
1448
1449         if (is->is_login_phase == false)
1450                 return (EBUSY);
1451
1452         if (is->is_terminating || is->is_reconnecting)
1453                 return (EIO);
1454
1455         datalen = ids->ids_data_segment_len;
1456         if (datalen > ISCSI_MAX_DATA_SEGMENT_LENGTH)
1457                 return (EINVAL);
1458         if (datalen > 0) {
1459                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1460                 error = copyin(ids->ids_data_segment, data, datalen);
1461                 if (error != 0) {
1462                         free(data, M_ISCSI);
1463                         return (error);
1464                 }
1465         }
1466
1467         ip = icl_pdu_new_bhs(is->is_conn, M_WAITOK);
1468         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1469         if (datalen > 0) {
1470                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1471                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1472                 free(data, M_ISCSI);
1473         }
1474         icl_pdu_queue(ip);
1475
1476         return (0);
1477 }
1478
1479 static int
1480 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1481     struct iscsi_daemon_receive *idr)
1482 {
1483         struct iscsi_session *is;
1484         struct icl_pdu *ip;
1485         void *data;
1486
1487         sx_slock(&sc->sc_lock);
1488         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1489                 if (is->is_id == idr->idr_session_id)
1490                         break;
1491         }
1492         if (is == NULL) {
1493                 sx_sunlock(&sc->sc_lock);
1494                 return (ESRCH);
1495         }
1496         sx_sunlock(&sc->sc_lock);
1497
1498         if (is->is_login_phase == false)
1499                 return (EBUSY);
1500
1501         ISCSI_SESSION_LOCK(is);
1502         while (is->is_login_pdu == NULL &&
1503             is->is_terminating == false &&
1504             is->is_reconnecting == false)
1505                 cv_wait(&is->is_login_cv, &is->is_lock);
1506         if (is->is_terminating || is->is_reconnecting) {
1507                 ISCSI_SESSION_UNLOCK(is);
1508                 return (EIO);
1509         }
1510         ip = is->is_login_pdu;
1511         is->is_login_pdu = NULL;
1512         ISCSI_SESSION_UNLOCK(is);
1513
1514         if (ip->ip_data_len > idr->idr_data_segment_len) {
1515                 icl_pdu_free(ip);
1516                 return (EMSGSIZE);
1517         }
1518
1519         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1520         if (ip->ip_data_len > 0) {
1521                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1522                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1523                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1524                 free(data, M_ISCSI);
1525         }
1526
1527         icl_pdu_free(ip);
1528
1529         return (0);
1530 }
1531
1532 static int
1533 iscsi_ioctl_daemon_close(struct iscsi_softc *sc,
1534     struct iscsi_daemon_close *idc)
1535 {
1536         struct iscsi_session *is;
1537
1538         sx_slock(&sc->sc_lock);
1539         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1540                 if (is->is_id == idc->idc_session_id)
1541                         break;
1542         }
1543         if (is == NULL) {
1544                 sx_sunlock(&sc->sc_lock);
1545                 return (ESRCH);
1546         }
1547         sx_sunlock(&sc->sc_lock);
1548
1549         iscsi_session_reconnect(is);
1550
1551         return (0);
1552 }
1553 #endif /* ICL_KERNEL_PROXY */
1554
1555 static void
1556 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1557 {
1558         /*
1559          * Just make sure all the fields are null-terminated.
1560          *
1561          * XXX: This is not particularly secure.  We should
1562          *      create our own conf and then copy in relevant
1563          *      fields.
1564          */
1565         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1566         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1567         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1568         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1569         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1570         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1571         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1572         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1573         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1574 }
1575
1576 static int
1577 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1578 {
1579         struct iscsi_session *is;
1580         const struct iscsi_session *is2;
1581         int error;
1582
1583         iscsi_sanitize_session_conf(&isa->isa_conf);
1584
1585         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1586         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1587
1588         if (is->is_conf.isc_initiator[0] == '\0' ||
1589             is->is_conf.isc_target_addr[0] == '\0') {
1590                 free(is, M_ISCSI);
1591                 return (EINVAL);
1592         }
1593
1594         if ((is->is_conf.isc_discovery != 0 && is->is_conf.isc_target[0] != 0) ||
1595             (is->is_conf.isc_discovery == 0 && is->is_conf.isc_target[0] == 0)) {
1596                 free(is, M_ISCSI);
1597                 return (EINVAL);
1598         }
1599
1600         sx_xlock(&sc->sc_lock);
1601
1602         /*
1603          * Prevent duplicates.
1604          */
1605         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1606                 if (!!is->is_conf.isc_discovery !=
1607                     !!is2->is_conf.isc_discovery)
1608                         continue;
1609
1610                 if (strcmp(is->is_conf.isc_target_addr,
1611                     is2->is_conf.isc_target_addr) != 0)
1612                         continue;
1613
1614                 if (is->is_conf.isc_discovery == 0 &&
1615                     strcmp(is->is_conf.isc_target,
1616                     is2->is_conf.isc_target) != 0)
1617                         continue;
1618
1619                 sx_xunlock(&sc->sc_lock);
1620                 free(is, M_ISCSI);
1621                 return (EBUSY);
1622         }
1623
1624         is->is_conn = icl_conn_new();
1625         is->is_conn->ic_receive = iscsi_receive_callback;
1626         is->is_conn->ic_error = iscsi_error_callback;
1627         is->is_conn->ic_prv0 = is;
1628         TAILQ_INIT(&is->is_outstanding);
1629         TAILQ_INIT(&is->is_postponed);
1630         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1631         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1632 #ifdef ICL_KERNEL_PROXY
1633         cv_init(&is->is_login_cv, "iscsi_login");
1634 #endif
1635
1636         is->is_softc = sc;
1637         sc->sc_last_session_id++;
1638         is->is_id = sc->sc_last_session_id;
1639         callout_init(&is->is_callout, 1);
1640         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1641         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1642
1643         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1644         if (error != 0) {
1645                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1646                 return (error);
1647         }
1648
1649         /*
1650          * Trigger immediate reconnection.
1651          */
1652         is->is_waiting_for_iscsid = true;
1653         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1654         cv_signal(&sc->sc_cv);
1655
1656         sx_xunlock(&sc->sc_lock);
1657
1658         return (0);
1659 }
1660
1661 static bool
1662 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1663     unsigned int id2, const struct iscsi_session_conf *c2)
1664 {
1665         if (id2 == 0 && c2->isc_target[0] == '\0' &&
1666             c2->isc_target_addr[0] == '\0')
1667                 return (true);
1668         if (id2 != 0 && id2 == id1)
1669                 return (true);
1670         if (c2->isc_target[0] != '\0' &&
1671             strcmp(c1->isc_target, c2->isc_target) == 0)
1672                 return (true);
1673         if (c2->isc_target_addr[0] != '\0' &&
1674             strcmp(c1->isc_target_addr, c2->isc_target_addr) == 0)
1675                 return (true);
1676         return (false);
1677 }
1678
1679 static int
1680 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1681     struct iscsi_session_remove *isr)
1682 {
1683         struct iscsi_session *is, *tmp;
1684         bool found = false;
1685
1686         iscsi_sanitize_session_conf(&isr->isr_conf);
1687
1688         sx_xlock(&sc->sc_lock);
1689         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1690                 ISCSI_SESSION_LOCK(is);
1691                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1692                     isr->isr_session_id, &isr->isr_conf)) {
1693                         found = true;
1694                         iscsi_session_logout(is);
1695                         iscsi_session_terminate(is);
1696                 }
1697                 ISCSI_SESSION_UNLOCK(is);
1698         }
1699         sx_xunlock(&sc->sc_lock);
1700
1701         if (!found)
1702                 return (ESRCH);
1703
1704         return (0);
1705 }
1706
1707 static int
1708 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1709 {
1710         int error;
1711         unsigned int i = 0;
1712         struct iscsi_session *is;
1713         struct iscsi_session_state iss;
1714
1715         sx_slock(&sc->sc_lock);
1716         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1717                 if (i >= isl->isl_nentries) {
1718                         sx_sunlock(&sc->sc_lock);
1719                         return (EMSGSIZE);
1720                 }
1721                 memset(&iss, 0, sizeof(iss));
1722                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1723                 iss.iss_id = is->is_id;
1724                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1725                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
1726
1727                 if (is->is_conn->ic_header_crc32c)
1728                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
1729                 else
1730                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
1731
1732                 if (is->is_conn->ic_data_crc32c)
1733                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
1734                 else
1735                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
1736
1737                 iss.iss_max_data_segment_length = is->is_max_data_segment_length;
1738                 iss.iss_immediate_data = is->is_immediate_data;
1739                 iss.iss_connected = is->is_connected;
1740
1741                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
1742                 if (error != 0) {
1743                         sx_sunlock(&sc->sc_lock);
1744                         return (error);
1745                 }
1746                 i++;
1747         }
1748         sx_sunlock(&sc->sc_lock);
1749
1750         isl->isl_nentries = i;
1751
1752         return (0);
1753 }
1754
1755 static int
1756 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
1757     struct thread *td)
1758 {
1759         struct iscsi_softc *sc;
1760
1761         sc = dev->si_drv1;
1762
1763         switch (cmd) {
1764         case ISCSIDWAIT:
1765                 return (iscsi_ioctl_daemon_wait(sc,
1766                     (struct iscsi_daemon_request *)arg));
1767         case ISCSIDHANDOFF:
1768                 return (iscsi_ioctl_daemon_handoff(sc,
1769                     (struct iscsi_daemon_handoff *)arg));
1770         case ISCSIDFAIL:
1771                 return (iscsi_ioctl_daemon_fail(sc,
1772                     (struct iscsi_daemon_fail *)arg));
1773 #ifdef ICL_KERNEL_PROXY
1774         case ISCSIDCONNECT:
1775                 return (iscsi_ioctl_daemon_connect(sc,
1776                     (struct iscsi_daemon_connect *)arg));
1777         case ISCSIDSEND:
1778                 return (iscsi_ioctl_daemon_send(sc,
1779                     (struct iscsi_daemon_send *)arg));
1780         case ISCSIDRECEIVE:
1781                 return (iscsi_ioctl_daemon_receive(sc,
1782                     (struct iscsi_daemon_receive *)arg));
1783         case ISCSIDCLOSE:
1784                 return (iscsi_ioctl_daemon_close(sc,
1785                     (struct iscsi_daemon_close *)arg));
1786 #endif /* ICL_KERNEL_PROXY */
1787         case ISCSISADD:
1788                 return (iscsi_ioctl_session_add(sc,
1789                     (struct iscsi_session_add *)arg));
1790         case ISCSISREMOVE:
1791                 return (iscsi_ioctl_session_remove(sc,
1792                     (struct iscsi_session_remove *)arg));
1793         case ISCSISLIST:
1794                 return (iscsi_ioctl_session_list(sc,
1795                     (struct iscsi_session_list *)arg));
1796         default:
1797                 return (EINVAL);
1798         }
1799 }
1800
1801 static uint64_t
1802 iscsi_encode_lun(uint32_t lun)
1803 {
1804         uint8_t encoded[8];
1805         uint64_t result;
1806
1807         memset(encoded, 0, sizeof(encoded));
1808
1809         if (lun < 256) {
1810                 /*
1811                  * Peripheral device addressing.
1812                  */
1813                 encoded[1] = lun;
1814         } else if (lun < 16384) {
1815                 /*
1816                  * Flat space addressing.
1817                  */
1818                 encoded[0] = 0x40;
1819                 encoded[0] |= (lun >> 8) & 0x3f;
1820                 encoded[1] = lun & 0xff;
1821         } else {
1822                 /*
1823                  * Extended flat space addressing.
1824                  */
1825                 encoded[0] = 0xd2;
1826                 encoded[1] = lun >> 16;
1827                 encoded[2] = lun >> 8;
1828                 encoded[3] = lun;
1829         }
1830
1831         memcpy(&result, encoded, sizeof(result));
1832         return (result);
1833 }
1834
1835 static struct iscsi_outstanding *
1836 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
1837 {
1838         struct iscsi_outstanding *io;
1839
1840         ISCSI_SESSION_LOCK_ASSERT(is);
1841
1842         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
1843                 if (io->io_initiator_task_tag == initiator_task_tag)
1844                         return (io);
1845         }
1846         return (NULL);
1847 }
1848
1849 static int
1850 iscsi_outstanding_add(struct iscsi_session *is,
1851     uint32_t initiator_task_tag, union ccb *ccb)
1852 {
1853         struct iscsi_outstanding *io;
1854
1855         ISCSI_SESSION_LOCK_ASSERT(is);
1856
1857         KASSERT(iscsi_outstanding_find(is, initiator_task_tag) == NULL,
1858             ("initiator_task_tag 0x%x already added", initiator_task_tag));
1859
1860         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
1861         if (io == NULL) {
1862                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes", sizeof(*io));
1863                 return (ENOMEM);
1864         }
1865         io->io_initiator_task_tag = initiator_task_tag;
1866         io->io_ccb = ccb;
1867         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
1868         return (0);
1869 }
1870
1871 static void
1872 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
1873 {
1874
1875         ISCSI_SESSION_LOCK_ASSERT(is);
1876
1877         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
1878         uma_zfree(iscsi_outstanding_zone, io);
1879 }
1880
1881 static void
1882 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
1883 {
1884         struct icl_pdu *request;
1885         struct iscsi_bhs_scsi_command *bhssc;
1886         struct ccb_scsiio *csio;
1887         size_t len;
1888         int error;
1889
1890         ISCSI_SESSION_LOCK_ASSERT(is);
1891
1892 #if 0
1893         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
1894 #else
1895         if (is->is_login_phase) {
1896                 ISCSI_SESSION_DEBUG(is, "called during login phase");
1897                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1898                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1899                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1900                 }
1901                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
1902                 xpt_done(ccb);
1903                 return;
1904         }
1905 #endif
1906
1907         request = icl_pdu_new_bhs(is->is_conn, M_NOWAIT);
1908         if (request == NULL) {
1909                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1910                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1911                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1912                 }
1913                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1914                 xpt_done(ccb);
1915                 return;
1916         }
1917
1918         csio = &ccb->csio;
1919         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
1920         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
1921         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
1922         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1923         case CAM_DIR_IN:
1924                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
1925                 break;
1926         case CAM_DIR_OUT:
1927                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
1928                 break;
1929         }
1930
1931         switch (csio->tag_action) {
1932         case MSG_HEAD_OF_Q_TAG:
1933                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
1934                 break;
1935                 break;
1936         case MSG_ORDERED_Q_TAG:
1937                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
1938                 break;
1939         case MSG_ACA_TASK:
1940                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
1941                 break;
1942         case CAM_TAG_ACTION_NONE:
1943         case MSG_SIMPLE_Q_TAG:
1944         default:
1945                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
1946                 break;
1947         }
1948
1949         bhssc->bhssc_lun = iscsi_encode_lun(csio->ccb_h.target_lun);
1950         bhssc->bhssc_initiator_task_tag = is->is_initiator_task_tag;
1951         is->is_initiator_task_tag++;
1952         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
1953         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
1954             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
1955
1956         if (csio->ccb_h.flags & CAM_CDB_POINTER)
1957                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
1958         else
1959                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
1960
1961         error = iscsi_outstanding_add(is, bhssc->bhssc_initiator_task_tag, ccb);
1962         if (error != 0) {
1963                 icl_pdu_free(request);
1964                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1965                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1966                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1967                 }
1968                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1969                 xpt_done(ccb);
1970                 return;
1971         }
1972
1973         if (is->is_immediate_data &&
1974             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
1975                 len = csio->dxfer_len;
1976                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
1977                 if (len > is->is_first_burst_length) {
1978                         ISCSI_SESSION_DEBUG(is, "len %zd -> %zd", len, is->is_first_burst_length);
1979                         len = is->is_first_burst_length;
1980                 }
1981
1982                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
1983                 if (error != 0) {
1984                         icl_pdu_free(request);
1985                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1986                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
1987                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
1988                         }
1989                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
1990                         xpt_done(ccb);
1991                         return;
1992                 }
1993         }
1994         iscsi_pdu_queue_locked(request);
1995 }
1996
1997 static void
1998 iscsi_action(struct cam_sim *sim, union ccb *ccb)
1999 {
2000         struct iscsi_session *is;
2001
2002         is = cam_sim_softc(sim);
2003
2004         ISCSI_SESSION_LOCK_ASSERT(is);
2005
2006         if (is->is_terminating) {
2007                 ISCSI_SESSION_DEBUG(is, "called during termination");
2008                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2009                 xpt_done(ccb);
2010                 return;
2011         }
2012
2013         switch (ccb->ccb_h.func_code) {
2014         case XPT_PATH_INQ:
2015         {
2016                 struct ccb_pathinq *cpi = &ccb->cpi;
2017
2018                 cpi->version_num = 1;
2019                 cpi->hba_inquiry = PI_TAG_ABLE;
2020                 cpi->target_sprt = 0;
2021                 //cpi->hba_misc = PIM_NOBUSRESET;
2022                 cpi->hba_misc = 0;
2023                 cpi->hba_eng_cnt = 0;
2024                 cpi->max_target = 0;
2025                 cpi->max_lun = 255;
2026                 //cpi->initiator_id = 0; /* XXX */
2027                 cpi->initiator_id = 64; /* XXX */
2028                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2029                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2030                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2031                 cpi->unit_number = cam_sim_unit(sim);
2032                 cpi->bus_id = cam_sim_bus(sim);
2033                 cpi->base_transfer_speed = 150000; /* XXX */
2034                 cpi->transport = XPORT_ISCSI;
2035                 cpi->transport_version = 0;
2036                 cpi->protocol = PROTO_SCSI;
2037                 cpi->protocol_version = SCSI_REV_SPC3;
2038                 cpi->maxio = MAXPHYS;
2039                 cpi->ccb_h.status = CAM_REQ_CMP;
2040                 break;
2041         }
2042         case XPT_CALC_GEOMETRY:
2043                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2044                 ccb->ccb_h.status = CAM_REQ_CMP;
2045                 break;
2046 #if 0
2047         /*
2048          * XXX: What's the point?
2049          */
2050         case XPT_RESET_BUS:
2051         case XPT_ABORT:
2052         case XPT_TERM_IO:
2053                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2054                 ccb->ccb_h.status = CAM_REQ_CMP;
2055                 break;
2056 #endif
2057         case XPT_SCSI_IO:
2058                 iscsi_action_scsiio(is, ccb);
2059                 return;
2060         default:
2061 #if 0
2062                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2063 #endif
2064                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2065                 break;
2066         }
2067         xpt_done(ccb);
2068 }
2069
2070 static void
2071 iscsi_poll(struct cam_sim *sim)
2072 {
2073
2074         KASSERT(0, ("%s: you're not supposed to be here", __func__));
2075 }
2076
2077 static void
2078 iscsi_shutdown(struct iscsi_softc *sc)
2079 {
2080         struct iscsi_session *is;
2081
2082         ISCSI_DEBUG("removing all sessions due to shutdown");
2083
2084         sx_slock(&sc->sc_lock);
2085         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2086                 iscsi_session_terminate(is);
2087         sx_sunlock(&sc->sc_lock);
2088 }
2089
2090 static int
2091 iscsi_load(void)
2092 {
2093         int error;
2094
2095         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2096         sx_init(&sc->sc_lock, "iscsi");
2097         TAILQ_INIT(&sc->sc_sessions);
2098         cv_init(&sc->sc_cv, "iscsi_cv");
2099
2100         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2101             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2102             UMA_ALIGN_PTR, 0);
2103
2104         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2105             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2106         if (error != 0) {
2107                 ISCSI_WARN("failed to create device node, error %d", error);
2108                 return (error);
2109         }
2110         sc->sc_cdev->si_drv1 = sc;
2111
2112         /*
2113          * Note that this needs to get run before dashutdown().  Otherwise,
2114          * when rebooting with iSCSI session with outstanding requests,
2115          * but disconnected, dashutdown() will hang on cam_periph_runccb().
2116          */
2117         sc->sc_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2118             iscsi_shutdown, sc, SHUTDOWN_PRI_FIRST);
2119
2120         return (0);
2121 }
2122
2123 static int
2124 iscsi_unload(void)
2125 {
2126         struct iscsi_session *is, *tmp;
2127
2128         if (sc->sc_cdev != NULL) {
2129                 ISCSI_DEBUG("removing device node");
2130                 destroy_dev(sc->sc_cdev);
2131                 ISCSI_DEBUG("device node removed");
2132         }
2133
2134         if (sc->sc_shutdown_eh != NULL)
2135                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_eh);
2136
2137         sx_slock(&sc->sc_lock);
2138         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp)
2139                 iscsi_session_terminate(is);
2140         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2141                 ISCSI_DEBUG("waiting for sessions to terminate");
2142                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2143         }
2144         ISCSI_DEBUG("all sessions terminated");
2145         sx_sunlock(&sc->sc_lock);
2146
2147         uma_zdestroy(iscsi_outstanding_zone);
2148         sx_destroy(&sc->sc_lock);
2149         cv_destroy(&sc->sc_cv);
2150         free(sc, M_ISCSI);
2151         return (0);
2152 }
2153
2154 static int
2155 iscsi_quiesce(void)
2156 {
2157         sx_slock(&sc->sc_lock);
2158         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2159                 sx_sunlock(&sc->sc_lock);
2160                 return (EBUSY);
2161         }
2162         sx_sunlock(&sc->sc_lock);
2163         return (0);
2164 }
2165
2166 static int
2167 iscsi_modevent(module_t mod, int what, void *arg)
2168 {
2169         int error;
2170
2171         switch (what) {
2172         case MOD_LOAD:
2173                 error = iscsi_load();
2174                 break;
2175         case MOD_UNLOAD:
2176                 error = iscsi_unload();
2177                 break;
2178         case MOD_QUIESCE:
2179                 error = iscsi_quiesce();
2180                 break;
2181         default:
2182                 error = EINVAL;
2183                 break;
2184         }
2185         return (error);
2186 }
2187
2188 moduledata_t iscsi_data = {
2189         "iscsi",
2190         iscsi_modevent,
2191         0
2192 };
2193
2194 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2195 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2196 MODULE_DEPEND(iscsi, icl, 1, 1, 1);