sys/cam/ctl/ctl_frontend_iscsi.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012 The FreeBSD Foundation
   5  * All rights reserved.
   6  *
   7  * This software was developed by Edward Tomasz Napierala under sponsorship
   8  * from the FreeBSD Foundation.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  * $FreeBSD$
  32  */
  33
  34 /*
  35  * CTL frontend for the iSCSI protocol.
  36  */
  37
  38 #include <sys/cdefs.h>
  39 __FBSDID("$FreeBSD$");
  40
  41 #include <sys/param.h>
  42 #include <sys/capsicum.h>
  43 #include <sys/condvar.h>
  44 #include <sys/endian.h>
  45 #include <sys/file.h>
  46 #include <sys/kernel.h>
  47 #include <sys/kthread.h>
  48 #include <sys/lock.h>
  49 #include <sys/malloc.h>
  50 #include <sys/module.h>
  51 #include <sys/mutex.h>
  52 #include <sys/queue.h>
  53 #include <sys/sbuf.h>
  54 #include <sys/socket.h>
  55 #include <sys/sysctl.h>
  56 #include <sys/systm.h>
  57 #include <sys/uio.h>
  58 #include <sys/unistd.h>
  59 #include <sys/nv.h>
  60 #include <sys/dnv.h>
  61 #include <vm/uma.h>
  62
  63 #include <cam/scsi/scsi_all.h>
  64 #include <cam/scsi/scsi_da.h>
  65 #include <cam/ctl/ctl_io.h>
  66 #include <cam/ctl/ctl.h>
  67 #include <cam/ctl/ctl_backend.h>
  68 #include <cam/ctl/ctl_error.h>
  69 #include <cam/ctl/ctl_frontend.h>
  70 #include <cam/ctl/ctl_debug.h>
  71 #include <cam/ctl/ctl_ha.h>
  72 #include <cam/ctl/ctl_ioctl.h>
  73 #include <cam/ctl/ctl_private.h>
  74
  75 #include <dev/iscsi/icl.h>
  76 #include <dev/iscsi/icl_wrappers.h>
  77 #include <dev/iscsi/iscsi_proto.h>
  78 #include <cam/ctl/ctl_frontend_iscsi.h>
  79
  80 #ifdef ICL_KERNEL_PROXY
  81 #include <sys/socketvar.h>
  82 #endif
  83
  84 #ifdef ICL_KERNEL_PROXY
  85 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
  86 #endif
  87
  88 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
  89 static uma_zone_t cfiscsi_data_wait_zone;
  90
  91 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
  92     "CAM Target Layer iSCSI Frontend");
  93 static int debug = 1;
  94 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  95     &debug, 1, "Enable debug messages");
  96 static int ping_timeout = 5;
  97 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
  98     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
  99 static int login_timeout = 60;
 100 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
 101     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
 102 static int maxtags = 256;
 103 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
 104     &maxtags, 0, "Max number of requests queued by initiator");
 105
 106 #define CFISCSI_DEBUG(X, ...)                                           \
 107         do {                                                            \
 108                 if (debug > 1) {                                        \
 109                         printf("%s: " X "\n",                           \
 110                             __func__, ## __VA_ARGS__);                  \
 111                 }                                                       \
 112         } while (0)
 113
 114 #define CFISCSI_WARN(X, ...)                                            \
 115         do {                                                            \
 116                 if (debug > 0) {                                        \
 117                         printf("WARNING: %s: " X "\n",                  \
 118                             __func__, ## __VA_ARGS__);                  \
 119                 }                                                       \
 120         } while (0)
 121
 122 #define CFISCSI_SESSION_DEBUG(S, X, ...)                                \
 123         do {                                                            \
 124                 if (debug > 1) {                                        \
 125                         printf("%s: %s (%s): " X "\n",                  \
 126                             __func__, S->cs_initiator_addr,             \
 127                             S->cs_initiator_name, ## __VA_ARGS__);      \
 128                 }                                                       \
 129         } while (0)
 130
 131 #define CFISCSI_SESSION_WARN(S, X, ...)                                 \
 132         do  {                                                           \
 133                 if (debug > 0) {                                        \
 134                         printf("WARNING: %s (%s): " X "\n",             \
 135                             S->cs_initiator_addr,                       \
 136                             S->cs_initiator_name, ## __VA_ARGS__);      \
 137                 }                                                       \
 138         } while (0)
 139
 140 #define CFISCSI_SESSION_LOCK(X)         mtx_lock(&X->cs_lock)
 141 #define CFISCSI_SESSION_UNLOCK(X)       mtx_unlock(&X->cs_lock)
 142 #define CFISCSI_SESSION_LOCK_ASSERT(X)  mtx_assert(&X->cs_lock, MA_OWNED)
 143
 144 #define CONN_SESSION(X)                 ((struct cfiscsi_session *)(X)->ic_prv0)
 145 #define PDU_SESSION(X)                  CONN_SESSION((X)->ip_conn)
 146 #define PDU_EXPDATASN(X)                (X)->ip_prv0
 147 #define PDU_TOTAL_TRANSFER_LEN(X)       (X)->ip_prv1
 148 #define PDU_R2TSN(X)                    (X)->ip_prv2
 149
 150 static int      cfiscsi_init(void);
 151 static int      cfiscsi_shutdown(void);
 152 static void     cfiscsi_online(void *arg);
 153 static void     cfiscsi_offline(void *arg);
 154 static int      cfiscsi_info(void *arg, struct sbuf *sb);
 155 static int      cfiscsi_ioctl(struct cdev *dev,
 156                     u_long cmd, caddr_t addr, int flag, struct thread *td);
 157 static void     cfiscsi_datamove(union ctl_io *io);
 158 static void     cfiscsi_datamove_in(union ctl_io *io);
 159 static void     cfiscsi_datamove_out(union ctl_io *io);
 160 static void     cfiscsi_done(union ctl_io *io);
 161 static bool     cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
 162 static void     cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
 163 static void     cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
 164 static void     cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
 165 static void     cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
 166 static void     cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
 167 static void     cfiscsi_session_terminate(struct cfiscsi_session *cs);
 168 static struct cfiscsi_data_wait *cfiscsi_data_wait_new(
 169                     struct cfiscsi_session *cs, union ctl_io *io,
 170                     uint32_t initiator_task_tag,
 171                     uint32_t *target_transfer_tagp);
 172 static void     cfiscsi_data_wait_free(struct cfiscsi_session *cs,
 173                     struct cfiscsi_data_wait *cdw);
 174 static struct cfiscsi_target    *cfiscsi_target_find(struct cfiscsi_softc
 175                     *softc, const char *name, uint16_t tag);
 176 static struct cfiscsi_target    *cfiscsi_target_find_or_create(
 177     struct cfiscsi_softc *softc, const char *name, const char *alias,
 178     uint16_t tag);
 179 static void     cfiscsi_target_release(struct cfiscsi_target *ct);
 180 static void     cfiscsi_session_delete(struct cfiscsi_session *cs);
 181
 182 static struct cfiscsi_softc cfiscsi_softc;
 183
 184 static struct ctl_frontend cfiscsi_frontend =
 185 {
 186         .name = "iscsi",
 187         .init = cfiscsi_init,
 188         .ioctl = cfiscsi_ioctl,
 189         .shutdown = cfiscsi_shutdown,
 190 };
 191 CTL_FRONTEND_DECLARE(cfiscsi, cfiscsi_frontend);
 192 MODULE_DEPEND(cfiscsi, icl, 1, 1, 1);
 193
 194 static struct icl_pdu *
 195 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
 196 {
 197
 198         return (icl_pdu_new(request->ip_conn, flags));
 199 }
 200
 201 static bool
 202 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
 203 {
 204         const struct iscsi_bhs_scsi_command *bhssc;
 205         struct cfiscsi_session *cs;
 206         uint32_t cmdsn, expstatsn;
 207
 208         cs = PDU_SESSION(request);
 209
 210         /*
 211          * Every incoming PDU - not just NOP-Out - resets the ping timer.
 212          * The purpose of the timeout is to reset the connection when it stalls;
 213          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 214          * in some queue.
 215          *
 216          * XXX: Locking?
 217          */
 218         cs->cs_timeout = 0;
 219
 220         /*
 221          * Data-Out PDUs don't contain CmdSN.
 222          */
 223         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 224             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 225                 return (false);
 226
 227         /*
 228          * We're only using fields common for all the request
 229          * (initiator -> target) PDUs.
 230          */
 231         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
 232         cmdsn = ntohl(bhssc->bhssc_cmdsn);
 233         expstatsn = ntohl(bhssc->bhssc_expstatsn);
 234
 235         CFISCSI_SESSION_LOCK(cs);
 236 #if 0
 237         if (expstatsn != cs->cs_statsn) {
 238                 CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
 239                     "while current StatSN is %d", expstatsn,
 240                     cs->cs_statsn);
 241         }
 242 #endif
 243
 244         if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
 245                 /*
 246                  * The target MUST silently ignore any non-immediate command
 247                  * outside of this range.
 248                  */
 249                 if (ISCSI_SNLT(cmdsn, cs->cs_cmdsn) ||
 250                     ISCSI_SNGT(cmdsn, cs->cs_cmdsn - 1 + maxtags)) {
 251                         CFISCSI_SESSION_UNLOCK(cs);
 252                         CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
 253                             "while expected %u", cmdsn, cs->cs_cmdsn);
 254                         return (true);
 255                 }
 256
 257                 /*
 258                  * We don't support multiple connections now, so any
 259                  * discontinuity in CmdSN means lost PDUs.  Since we don't
 260                  * support PDU retransmission -- terminate the connection.
 261                  */
 262                 if (cmdsn != cs->cs_cmdsn) {
 263                         CFISCSI_SESSION_UNLOCK(cs);
 264                         CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
 265                             "while expected %u; dropping connection",
 266                             cmdsn, cs->cs_cmdsn);
 267                         cfiscsi_session_terminate(cs);
 268                         return (true);
 269                 }
 270                 cs->cs_cmdsn++;
 271         }
 272
 273         CFISCSI_SESSION_UNLOCK(cs);
 274
 275         return (false);
 276 }
 277
 278 static void
 279 cfiscsi_pdu_handle(struct icl_pdu *request)
 280 {
 281         struct cfiscsi_session *cs;
 282         bool ignore;
 283
 284         cs = PDU_SESSION(request);
 285
 286         ignore = cfiscsi_pdu_update_cmdsn(request);
 287         if (ignore) {
 288                 icl_pdu_free(request);
 289                 return;
 290         }
 291
 292         /*
 293          * Handle the PDU; this includes e.g. receiving the remaining
 294          * part of PDU and submitting the SCSI command to CTL
 295          * or queueing a reply.  The handling routine is responsible
 296          * for freeing the PDU when it's no longer needed.
 297          */
 298         switch (request->ip_bhs->bhs_opcode &
 299             ~ISCSI_BHS_OPCODE_IMMEDIATE) {
 300         case ISCSI_BHS_OPCODE_NOP_OUT:
 301                 cfiscsi_pdu_handle_nop_out(request);
 302                 break;
 303         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
 304                 cfiscsi_pdu_handle_scsi_command(request);
 305                 break;
 306         case ISCSI_BHS_OPCODE_TASK_REQUEST:
 307                 cfiscsi_pdu_handle_task_request(request);
 308                 break;
 309         case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
 310                 cfiscsi_pdu_handle_data_out(request);
 311                 break;
 312         case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
 313                 cfiscsi_pdu_handle_logout_request(request);
 314                 break;
 315         default:
 316                 CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
 317                     "opcode 0x%x; dropping connection",
 318                     request->ip_bhs->bhs_opcode);
 319                 icl_pdu_free(request);
 320                 cfiscsi_session_terminate(cs);
 321         }
 322
 323 }
 324
 325 static void
 326 cfiscsi_receive_callback(struct icl_pdu *request)
 327 {
 328 #ifdef ICL_KERNEL_PROXY
 329         struct cfiscsi_session *cs;
 330
 331         cs = PDU_SESSION(request);
 332         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 333                 if (cs->cs_login_pdu == NULL)
 334                         cs->cs_login_pdu = request;
 335                 else
 336                         icl_pdu_free(request);
 337                 cv_signal(&cs->cs_login_cv);
 338                 return;
 339         }
 340 #endif
 341
 342         cfiscsi_pdu_handle(request);
 343 }
 344
 345 static void
 346 cfiscsi_error_callback(struct icl_conn *ic)
 347 {
 348         struct cfiscsi_session *cs;
 349
 350         cs = CONN_SESSION(ic);
 351
 352         CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
 353         cfiscsi_session_terminate(cs);
 354 }
 355
 356 static int
 357 cfiscsi_pdu_prepare(struct icl_pdu *response)
 358 {
 359         struct cfiscsi_session *cs;
 360         struct iscsi_bhs_scsi_response *bhssr;
 361         bool advance_statsn = true;
 362
 363         cs = PDU_SESSION(response);
 364
 365         CFISCSI_SESSION_LOCK_ASSERT(cs);
 366
 367         /*
 368          * We're only using fields common for all the response
 369          * (target -> initiator) PDUs.
 370          */
 371         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 372
 373         /*
 374          * 10.8.3: "The StatSN for this connection is not advanced
 375          * after this PDU is sent."
 376          */
 377         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
 378                 advance_statsn = false;
 379
 380         /*
 381          * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
 382          * StatSN for the connection is not advanced after this PDU is sent."
 383          */
 384         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
 385             bhssr->bhssr_initiator_task_tag == 0xffffffff)
 386                 advance_statsn = false;
 387
 388         /*
 389          * See the comment below - StatSN is not meaningful and must
 390          * not be advanced.
 391          */
 392         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
 393             (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
 394                 advance_statsn = false;
 395
 396         /*
 397          * 10.7.3: "The fields StatSN, Status, and Residual Count
 398          * only have meaningful content if the S bit is set to 1."
 399          */
 400         if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 401             (bhssr->bhssr_flags & BHSDI_FLAGS_S))
 402                 bhssr->bhssr_statsn = htonl(cs->cs_statsn);
 403         bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
 404         bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn - 1 +
 405             imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
 406
 407         if (advance_statsn)
 408                 cs->cs_statsn++;
 409
 410         return (0);
 411 }
 412
 413 static void
 414 cfiscsi_pdu_queue(struct icl_pdu *response)
 415 {
 416         struct cfiscsi_session *cs;
 417
 418         cs = PDU_SESSION(response);
 419
 420         CFISCSI_SESSION_LOCK(cs);
 421         cfiscsi_pdu_prepare(response);
 422         icl_pdu_queue(response);
 423         CFISCSI_SESSION_UNLOCK(cs);
 424 }
 425
 426 static void
 427 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
 428 {
 429         struct cfiscsi_session *cs;
 430         struct iscsi_bhs_nop_out *bhsno;
 431         struct iscsi_bhs_nop_in *bhsni;
 432         struct icl_pdu *response;
 433         void *data = NULL;
 434         size_t datasize;
 435         int error;
 436
 437         cs = PDU_SESSION(request);
 438         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 439
 440         if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
 441                 /*
 442                  * Nothing to do, iscsi_pdu_update_statsn() already
 443                  * zeroed the timeout.
 444                  */
 445                 icl_pdu_free(request);
 446                 return;
 447         }
 448
 449         datasize = icl_pdu_data_segment_length(request);
 450         if (datasize > 0) {
 451                 data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
 452                 if (data == NULL) {
 453                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 454                             "dropping connection");
 455                         icl_pdu_free(request);
 456                         cfiscsi_session_terminate(cs);
 457                         return;
 458                 }
 459                 icl_pdu_get_data(request, 0, data, datasize);
 460         }
 461
 462         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 463         if (response == NULL) {
 464                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 465                     "droppping connection");
 466                 free(data, M_CFISCSI);
 467                 icl_pdu_free(request);
 468                 cfiscsi_session_terminate(cs);
 469                 return;
 470         }
 471         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 472         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
 473         bhsni->bhsni_flags = 0x80;
 474         bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
 475         bhsni->bhsni_target_transfer_tag = 0xffffffff;
 476         if (datasize > 0) {
 477                 error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
 478                 if (error != 0) {
 479                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 480                             "dropping connection");
 481                         free(data, M_CFISCSI);
 482                         icl_pdu_free(request);
 483                         icl_pdu_free(response);
 484                         cfiscsi_session_terminate(cs);
 485                         return;
 486                 }
 487                 free(data, M_CFISCSI);
 488         }
 489
 490         icl_pdu_free(request);
 491         cfiscsi_pdu_queue(response);
 492 }
 493
 494 static void
 495 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
 496 {
 497         struct iscsi_bhs_scsi_command *bhssc;
 498         struct cfiscsi_session *cs;
 499         union ctl_io *io;
 500         int error;
 501
 502         cs = PDU_SESSION(request);
 503         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 504         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
 505         //    bhssc->bhssc_initiator_task_tag);
 506
 507         if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
 508                 CFISCSI_SESSION_WARN(cs, "unsolicited data with "
 509                     "ImmediateData=No; dropping connection");
 510                 icl_pdu_free(request);
 511                 cfiscsi_session_terminate(cs);
 512                 return;
 513         }
 514         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
 515         ctl_zero_io(io);
 516         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 517         io->io_hdr.io_type = CTL_IO_SCSI;
 518         io->io_hdr.nexus.initid = cs->cs_ctl_initid;
 519         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
 520         io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
 521         io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
 522         switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
 523         case BHSSC_FLAGS_ATTR_UNTAGGED:
 524                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 525                 break;
 526         case BHSSC_FLAGS_ATTR_SIMPLE:
 527                 io->scsiio.tag_type = CTL_TAG_SIMPLE;
 528                 break;
 529         case BHSSC_FLAGS_ATTR_ORDERED:
 530                 io->scsiio.tag_type = CTL_TAG_ORDERED;
 531                 break;
 532         case BHSSC_FLAGS_ATTR_HOQ:
 533                 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
 534                 break;
 535         case BHSSC_FLAGS_ATTR_ACA:
 536                 io->scsiio.tag_type = CTL_TAG_ACA;
 537                 break;
 538         default:
 539                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 540                 CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
 541                     bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
 542                 break;
 543         }
 544         io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
 545         memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
 546         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 547         error = ctl_queue(io);
 548         if (error != CTL_RETVAL_COMPLETE) {
 549                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 550                     "dropping connection", error);
 551                 ctl_free_io(io);
 552                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 553                 icl_pdu_free(request);
 554                 cfiscsi_session_terminate(cs);
 555         }
 556 }
 557
 558 static void
 559 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
 560 {
 561         struct iscsi_bhs_task_management_request *bhstmr;
 562         struct iscsi_bhs_task_management_response *bhstmr2;
 563         struct icl_pdu *response;
 564         struct cfiscsi_session *cs;
 565         union ctl_io *io;
 566         int error;
 567
 568         cs = PDU_SESSION(request);
 569         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
 570         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
 571         ctl_zero_io(io);
 572         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 573         io->io_hdr.io_type = CTL_IO_TASK;
 574         io->io_hdr.nexus.initid = cs->cs_ctl_initid;
 575         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
 576         io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
 577         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
 578
 579         switch (bhstmr->bhstmr_function & ~0x80) {
 580         case BHSTMR_FUNCTION_ABORT_TASK:
 581 #if 0
 582                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
 583 #endif
 584                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
 585                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
 586                 break;
 587         case BHSTMR_FUNCTION_ABORT_TASK_SET:
 588 #if 0
 589                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
 590 #endif
 591                 io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
 592                 break;
 593         case BHSTMR_FUNCTION_CLEAR_TASK_SET:
 594 #if 0
 595                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
 596 #endif
 597                 io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
 598                 break;
 599         case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
 600 #if 0
 601                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
 602 #endif
 603                 io->taskio.task_action = CTL_TASK_LUN_RESET;
 604                 break;
 605         case BHSTMR_FUNCTION_TARGET_WARM_RESET:
 606 #if 0
 607                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
 608 #endif
 609                 io->taskio.task_action = CTL_TASK_TARGET_RESET;
 610                 break;
 611         case BHSTMR_FUNCTION_TARGET_COLD_RESET:
 612 #if 0
 613                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
 614 #endif
 615                 io->taskio.task_action = CTL_TASK_TARGET_RESET;
 616                 break;
 617         case BHSTMR_FUNCTION_QUERY_TASK:
 618 #if 0
 619                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
 620 #endif
 621                 io->taskio.task_action = CTL_TASK_QUERY_TASK;
 622                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
 623                 break;
 624         case BHSTMR_FUNCTION_QUERY_TASK_SET:
 625 #if 0
 626                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
 627 #endif
 628                 io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
 629                 break;
 630         case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
 631 #if 0
 632                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
 633 #endif
 634                 io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
 635                 break;
 636         case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
 637 #if 0
 638                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
 639 #endif
 640                 io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
 641                 break;
 642         default:
 643                 CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
 644                     bhstmr->bhstmr_function & ~0x80);
 645                 ctl_free_io(io);
 646
 647                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 648                 if (response == NULL) {
 649                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 650                             "dropping connection");
 651                         icl_pdu_free(request);
 652                         cfiscsi_session_terminate(cs);
 653                         return;
 654                 }
 655                 bhstmr2 = (struct iscsi_bhs_task_management_response *)
 656                     response->ip_bhs;
 657                 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
 658                 bhstmr2->bhstmr_flags = 0x80;
 659                 bhstmr2->bhstmr_response =
 660                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
 661                 bhstmr2->bhstmr_initiator_task_tag =
 662                     bhstmr->bhstmr_initiator_task_tag;
 663                 icl_pdu_free(request);
 664                 cfiscsi_pdu_queue(response);
 665                 return;
 666         }
 667
 668         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 669         error = ctl_queue(io);
 670         if (error != CTL_RETVAL_COMPLETE) {
 671                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 672                     "dropping connection", error);
 673                 ctl_free_io(io);
 674                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 675                 icl_pdu_free(request);
 676                 cfiscsi_session_terminate(cs);
 677         }
 678 }
 679
 680 static bool
 681 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
 682 {
 683         struct iscsi_bhs_data_out *bhsdo;
 684         struct cfiscsi_session *cs;
 685         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
 686         size_t copy_len, len, off, buffer_offset;
 687         int ctl_sg_count;
 688         union ctl_io *io;
 689
 690         cs = PDU_SESSION(request);
 691
 692         KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 693             ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
 694             (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 695             ISCSI_BHS_OPCODE_SCSI_COMMAND,
 696             ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
 697
 698         /*
 699          * We're only using fields common for Data-Out and SCSI Command PDUs.
 700          */
 701         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 702
 703         io = cdw->cdw_ctl_io;
 704         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 705             ("CTL_FLAG_DATA_IN"));
 706
 707 #if 0
 708         CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
 709             request->ip_data_len, io->scsiio.kern_total_len);
 710 #endif
 711
 712         if (io->scsiio.kern_sg_entries > 0) {
 713                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
 714                 ctl_sg_count = io->scsiio.kern_sg_entries;
 715         } else {
 716                 ctl_sglist = &ctl_sg_entry;
 717                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
 718                 ctl_sglist->len = io->scsiio.kern_data_len;
 719                 ctl_sg_count = 1;
 720         }
 721
 722         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 723             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 724                 buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
 725         else
 726                 buffer_offset = 0;
 727         len = icl_pdu_data_segment_length(request);
 728
 729         /*
 730          * Make sure the offset, as sent by the initiator, matches the offset
 731          * we're supposed to be at in the scatter-gather list.
 732          */
 733         if (buffer_offset >
 734             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
 735             buffer_offset + len <=
 736             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
 737                 CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
 738                     "expected %zd; dropping connection", buffer_offset,
 739                     (size_t)io->scsiio.kern_rel_offset +
 740                     (size_t)io->scsiio.ext_data_filled);
 741                 ctl_set_data_phase_error(&io->scsiio);
 742                 cfiscsi_session_terminate(cs);
 743                 return (true);
 744         }
 745
 746         /*
 747          * This is the offset within the PDU data segment, as opposed
 748          * to buffer_offset, which is the offset within the task (SCSI
 749          * command).
 750          */
 751         off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
 752             buffer_offset;
 753
 754         /*
 755          * Iterate over the scatter/gather segments, filling them with data
 756          * from the PDU data segment.  Note that this can get called multiple
 757          * times for one SCSI command; the cdw structure holds state for the
 758          * scatter/gather list.
 759          */
 760         for (;;) {
 761                 KASSERT(cdw->cdw_sg_index < ctl_sg_count,
 762                     ("cdw->cdw_sg_index >= ctl_sg_count"));
 763                 if (cdw->cdw_sg_len == 0) {
 764                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
 765                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
 766                 }
 767                 KASSERT(off <= len, ("len > off"));
 768                 copy_len = len - off;
 769                 if (copy_len > cdw->cdw_sg_len)
 770                         copy_len = cdw->cdw_sg_len;
 771
 772                 icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
 773                 cdw->cdw_sg_addr += copy_len;
 774                 cdw->cdw_sg_len -= copy_len;
 775                 off += copy_len;
 776                 io->scsiio.ext_data_filled += copy_len;
 777                 io->scsiio.kern_data_resid -= copy_len;
 778
 779                 if (cdw->cdw_sg_len == 0) {
 780                         /*
 781                          * End of current segment.
 782                          */
 783                         if (cdw->cdw_sg_index == ctl_sg_count - 1) {
 784                                 /*
 785                                  * Last segment in scatter/gather list.
 786                                  */
 787                                 break;
 788                         }
 789                         cdw->cdw_sg_index++;
 790                 }
 791
 792                 if (off == len) {
 793                         /*
 794                          * End of PDU payload.
 795                          */
 796                         break;
 797                 }
 798         }
 799
 800         if (len > off) {
 801                 /*
 802                  * In case of unsolicited data, it's possible that the buffer
 803                  * provided by CTL is smaller than negotiated FirstBurstLength.
 804                  * Just ignore the superfluous data; will ask for them with R2T
 805                  * on next call to cfiscsi_datamove().
 806                  *
 807                  * This obviously can only happen with SCSI Command PDU.
 808                  */
 809                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 810                     ISCSI_BHS_OPCODE_SCSI_COMMAND)
 811                         return (true);
 812
 813                 CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
 814                     "expected %zd; dropping connection",
 815                     icl_pdu_data_segment_length(request), off);
 816                 ctl_set_data_phase_error(&io->scsiio);
 817                 cfiscsi_session_terminate(cs);
 818                 return (true);
 819         }
 820
 821         if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
 822             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
 823                 CFISCSI_SESSION_WARN(cs, "got the final packet without "
 824                     "the F flag; flags = 0x%x; dropping connection",
 825                     bhsdo->bhsdo_flags);
 826                 ctl_set_data_phase_error(&io->scsiio);
 827                 cfiscsi_session_terminate(cs);
 828                 return (true);
 829         }
 830
 831         if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
 832             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
 833                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 834                     ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 835                         CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
 836                             "transmitted size was %zd bytes instead of %d; "
 837                             "dropping connection",
 838                             (size_t)io->scsiio.ext_data_filled,
 839                             cdw->cdw_r2t_end);
 840                         ctl_set_data_phase_error(&io->scsiio);
 841                         cfiscsi_session_terminate(cs);
 842                         return (true);
 843                 } else {
 844                         /*
 845                          * For SCSI Command PDU, this just means we need to
 846                          * solicit more data by sending R2T.
 847                          */
 848                         return (false);
 849                 }
 850         }
 851
 852         if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
 853 #if 0
 854                 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
 855                     "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
 856 #endif
 857
 858                 return (true);
 859         }
 860
 861         return (false);
 862 }
 863
 864 static void
 865 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
 866 {
 867         struct iscsi_bhs_data_out *bhsdo;
 868         struct cfiscsi_session *cs;
 869         struct cfiscsi_data_wait *cdw = NULL;
 870         union ctl_io *io;
 871         bool done;
 872
 873         cs = PDU_SESSION(request);
 874         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 875
 876         CFISCSI_SESSION_LOCK(cs);
 877         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
 878 #if 0
 879                 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
 880                     "ttt 0x%x, itt 0x%x",
 881                     bhsdo->bhsdo_target_transfer_tag,
 882                     bhsdo->bhsdo_initiator_task_tag,
 883                     cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
 884 #endif
 885                 if (bhsdo->bhsdo_target_transfer_tag ==
 886                     cdw->cdw_target_transfer_tag)
 887                         break;
 888         }
 889         CFISCSI_SESSION_UNLOCK(cs);
 890         if (cdw == NULL) {
 891                 CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
 892                     "0x%x, not found; dropping connection",
 893                     bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
 894                 icl_pdu_free(request);
 895                 cfiscsi_session_terminate(cs);
 896                 return;
 897         }
 898
 899         if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
 900                 CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
 901                     "DataSN %u, while expected %u; dropping connection",
 902                     ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
 903                 icl_pdu_free(request);
 904                 cfiscsi_session_terminate(cs);
 905                 return;
 906         }
 907         cdw->cdw_datasn++;
 908
 909         io = cdw->cdw_ctl_io;
 910         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 911             ("CTL_FLAG_DATA_IN"));
 912
 913         done = cfiscsi_handle_data_segment(request, cdw);
 914         if (done) {
 915                 CFISCSI_SESSION_LOCK(cs);
 916                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
 917                 CFISCSI_SESSION_UNLOCK(cs);
 918                 done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
 919                     io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
 920                 cfiscsi_data_wait_free(cs, cdw);
 921                 io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
 922                 if (done)
 923                         io->scsiio.be_move_done(io);
 924                 else
 925                         cfiscsi_datamove_out(io);
 926         }
 927
 928         icl_pdu_free(request);
 929 }
 930
 931 static void
 932 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
 933 {
 934         struct iscsi_bhs_logout_request *bhslr;
 935         struct iscsi_bhs_logout_response *bhslr2;
 936         struct icl_pdu *response;
 937         struct cfiscsi_session *cs;
 938
 939         cs = PDU_SESSION(request);
 940         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 941         switch (bhslr->bhslr_reason & 0x7f) {
 942         case BHSLR_REASON_CLOSE_SESSION:
 943         case BHSLR_REASON_CLOSE_CONNECTION:
 944                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 945                 if (response == NULL) {
 946                         CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
 947                         icl_pdu_free(request);
 948                         cfiscsi_session_terminate(cs);
 949                         return;
 950                 }
 951                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 952                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 953                 bhslr2->bhslr_flags = 0x80;
 954                 bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
 955                 bhslr2->bhslr_initiator_task_tag =
 956                     bhslr->bhslr_initiator_task_tag;
 957                 icl_pdu_free(request);
 958                 cfiscsi_pdu_queue(response);
 959                 cfiscsi_session_terminate(cs);
 960                 break;
 961         case BHSLR_REASON_REMOVE_FOR_RECOVERY:
 962                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 963                 if (response == NULL) {
 964                         CFISCSI_SESSION_WARN(cs,
 965                             "failed to allocate memory; dropping connection");
 966                         icl_pdu_free(request);
 967                         cfiscsi_session_terminate(cs);
 968                         return;
 969                 }
 970                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 971                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 972                 bhslr2->bhslr_flags = 0x80;
 973                 bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
 974                 bhslr2->bhslr_initiator_task_tag =
 975                     bhslr->bhslr_initiator_task_tag;
 976                 icl_pdu_free(request);
 977                 cfiscsi_pdu_queue(response);
 978                 break;
 979         default:
 980                 CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
 981                     bhslr->bhslr_reason);
 982                 icl_pdu_free(request);
 983                 cfiscsi_session_terminate(cs);
 984                 break;
 985         }
 986 }
 987
 988 static void
 989 cfiscsi_callout(void *context)
 990 {
 991         struct icl_pdu *cp;
 992         struct iscsi_bhs_nop_in *bhsni;
 993         struct cfiscsi_session *cs;
 994
 995         cs = context;
 996
 997         if (cs->cs_terminating)
 998                 return;
 999
1000         callout_schedule(&cs->cs_callout, 1 * hz);
1001
1002         atomic_add_int(&cs->cs_timeout, 1);
1003
1004 #ifdef ICL_KERNEL_PROXY
1005         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1006                 if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1007                         CFISCSI_SESSION_WARN(cs, "login timed out after "
1008                             "%d seconds; dropping connection", cs->cs_timeout);
1009                         cfiscsi_session_terminate(cs);
1010                 }
1011                 return;
1012         }
1013 #endif
1014
1015         if (ping_timeout <= 0) {
1016                 /*
1017                  * Pings are disabled.  Don't send NOP-In in this case;
1018                  * user might have disabled pings to work around problems
1019                  * with certain initiators that can't properly handle
1020                  * NOP-In, such as iPXE.  Reset the timeout, to avoid
1021                  * triggering reconnection, should the user decide to
1022                  * reenable them.
1023                  */
1024                 cs->cs_timeout = 0;
1025                 return;
1026         }
1027
1028         if (cs->cs_timeout >= ping_timeout) {
1029                 CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1030                     "dropping connection",  ping_timeout);
1031                 cfiscsi_session_terminate(cs);
1032                 return;
1033         }
1034
1035         /*
1036          * If the ping was reset less than one second ago - which means
1037          * that we've received some PDU during the last second - assume
1038          * the traffic flows correctly and don't bother sending a NOP-Out.
1039          *
1040          * (It's 2 - one for one second, and one for incrementing is_timeout
1041          * earlier in this routine.)
1042          */
1043         if (cs->cs_timeout < 2)
1044                 return;
1045
1046         cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1047         if (cp == NULL) {
1048                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1049                 return;
1050         }
1051         bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1052         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1053         bhsni->bhsni_flags = 0x80;
1054         bhsni->bhsni_initiator_task_tag = 0xffffffff;
1055
1056         cfiscsi_pdu_queue(cp);
1057 }
1058
1059 static struct cfiscsi_data_wait *
1060 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1061     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1062 {
1063         struct cfiscsi_data_wait *cdw;
1064         int error;
1065
1066         cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1067         if (cdw == NULL) {
1068                 CFISCSI_SESSION_WARN(cs,
1069                     "failed to allocate %zd bytes", sizeof(*cdw));
1070                 return (NULL);
1071         }
1072
1073         error = icl_conn_transfer_setup(cs->cs_conn, io, target_transfer_tagp,
1074             &cdw->cdw_icl_prv);
1075         if (error != 0) {
1076                 CFISCSI_SESSION_WARN(cs,
1077                     "icl_conn_transfer_setup() failed with error %d", error);
1078                 uma_zfree(cfiscsi_data_wait_zone, cdw);
1079                 return (NULL);
1080         }
1081
1082         cdw->cdw_ctl_io = io;
1083         cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1084         cdw->cdw_initiator_task_tag = initiator_task_tag;
1085
1086         return (cdw);
1087 }
1088
1089 static void
1090 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1091     struct cfiscsi_data_wait *cdw)
1092 {
1093
1094         icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1095         uma_zfree(cfiscsi_data_wait_zone, cdw);
1096 }
1097
1098 static void
1099 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1100 {
1101         struct cfiscsi_data_wait *cdw;
1102         union ctl_io *io;
1103         int error, last, wait;
1104
1105         if (cs->cs_target == NULL)
1106                 return;         /* No target yet, so nothing to do. */
1107         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1108         ctl_zero_io(io);
1109         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs;
1110         io->io_hdr.io_type = CTL_IO_TASK;
1111         io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1112         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1113         io->io_hdr.nexus.targ_lun = 0;
1114         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1115         io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1116         wait = cs->cs_outstanding_ctl_pdus;
1117         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1118         error = ctl_queue(io);
1119         if (error != CTL_RETVAL_COMPLETE) {
1120                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1121                 refcount_release(&cs->cs_outstanding_ctl_pdus);
1122                 ctl_free_io(io);
1123         }
1124
1125         CFISCSI_SESSION_LOCK(cs);
1126         while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1127                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1128                 CFISCSI_SESSION_UNLOCK(cs);
1129                 /*
1130                  * Set nonzero port status; this prevents backends from
1131                  * assuming that the data transfer actually succeeded
1132                  * and writing uninitialized data to disk.
1133                  */
1134                 cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1135                 cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1136                 cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1137                 cfiscsi_data_wait_free(cs, cdw);
1138                 CFISCSI_SESSION_LOCK(cs);
1139         }
1140         CFISCSI_SESSION_UNLOCK(cs);
1141
1142         /*
1143          * Wait for CTL to terminate all the tasks.
1144          */
1145         if (wait > 0)
1146                 CFISCSI_SESSION_WARN(cs,
1147                     "waiting for CTL to terminate %d tasks", wait);
1148         for (;;) {
1149                 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1150                 last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1151                 if (last != 0)
1152                         break;
1153                 tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1154                     0, "cfiscsi_terminate", hz / 100);
1155         }
1156         if (wait > 0)
1157                 CFISCSI_SESSION_WARN(cs, "tasks terminated");
1158 }
1159
1160 static void
1161 cfiscsi_maintenance_thread(void *arg)
1162 {
1163         struct cfiscsi_session *cs;
1164
1165         cs = arg;
1166
1167         for (;;) {
1168                 CFISCSI_SESSION_LOCK(cs);
1169                 if (cs->cs_terminating == false || cs->cs_handoff_in_progress)
1170                         cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1171                 CFISCSI_SESSION_UNLOCK(cs);
1172
1173                 if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {
1174
1175                         /*
1176                          * We used to wait up to 30 seconds to deliver queued
1177                          * PDUs to the initiator.  We also tried hard to deliver
1178                          * SCSI Responses for the aborted PDUs.  We don't do
1179                          * that anymore.  We might need to revisit that.
1180                          */
1181                         callout_drain(&cs->cs_callout);
1182                         icl_conn_close(cs->cs_conn);
1183
1184                         /*
1185                          * At this point ICL receive thread is no longer
1186                          * running; no new tasks can be queued.
1187                          */
1188                         cfiscsi_session_terminate_tasks(cs);
1189                         cfiscsi_session_delete(cs);
1190                         kthread_exit();
1191                         return;
1192                 }
1193                 CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1194         }
1195 }
1196
1197 static void
1198 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1199 {
1200
1201         cs->cs_terminating = true;
1202         cv_signal(&cs->cs_maintenance_cv);
1203 #ifdef ICL_KERNEL_PROXY
1204         cv_signal(&cs->cs_login_cv);
1205 #endif
1206 }
1207
1208 static int
1209 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1210 {
1211         struct cfiscsi_target *ct;
1212         char *name;
1213         int i;
1214
1215         KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1216
1217         ct = cs->cs_target;
1218         name = strdup(cs->cs_initiator_id, M_CTL);
1219         i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1220         if (i < 0) {
1221                 CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1222                     i);
1223                 cs->cs_ctl_initid = -1;
1224                 return (1);
1225         }
1226         cs->cs_ctl_initid = i;
1227 #if 0
1228         CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1229 #endif
1230
1231         return (0);
1232 }
1233
1234 static void
1235 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1236 {
1237         int error;
1238
1239         if (cs->cs_ctl_initid == -1)
1240                 return;
1241
1242         error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1243         if (error != 0) {
1244                 CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1245                     error);
1246         }
1247         cs->cs_ctl_initid = -1;
1248 }
1249
1250 static struct cfiscsi_session *
1251 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1252 {
1253         struct cfiscsi_session *cs;
1254         int error;
1255
1256         cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1257         if (cs == NULL) {
1258                 CFISCSI_WARN("malloc failed");
1259                 return (NULL);
1260         }
1261         cs->cs_ctl_initid = -1;
1262
1263         refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1264         TAILQ_INIT(&cs->cs_waiting_for_data_out);
1265         mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1266         cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1267 #ifdef ICL_KERNEL_PROXY
1268         cv_init(&cs->cs_login_cv, "cfiscsi_login");
1269 #endif
1270
1271         /*
1272          * The purpose of this is to avoid racing with session shutdown.
1273          * Otherwise we could have the maintenance thread call icl_conn_close()
1274          * before we call icl_conn_handoff().
1275          */
1276         cs->cs_handoff_in_progress = true;
1277
1278         cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1279         if (cs->cs_conn == NULL) {
1280                 free(cs, M_CFISCSI);
1281                 return (NULL);
1282         }
1283         cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1284         cs->cs_conn->ic_error = cfiscsi_error_callback;
1285         cs->cs_conn->ic_prv0 = cs;
1286
1287         error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1288         if (error != 0) {
1289                 CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1290                 free(cs, M_CFISCSI);
1291                 return (NULL);
1292         }
1293
1294         mtx_lock(&softc->lock);
1295         cs->cs_id = ++softc->last_session_id;
1296         TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1297         mtx_unlock(&softc->lock);
1298
1299         /*
1300          * Start pinging the initiator.
1301          */
1302         callout_init(&cs->cs_callout, 1);
1303         callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1304
1305         return (cs);
1306 }
1307
1308 static void
1309 cfiscsi_session_delete(struct cfiscsi_session *cs)
1310 {
1311         struct cfiscsi_softc *softc;
1312
1313         softc = &cfiscsi_softc;
1314
1315         KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1316             ("destroying session with outstanding CTL pdus"));
1317         KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1318             ("destroying session with non-empty queue"));
1319
1320         mtx_lock(&softc->lock);
1321         TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1322         mtx_unlock(&softc->lock);
1323
1324         cfiscsi_session_unregister_initiator(cs);
1325         if (cs->cs_target != NULL)
1326                 cfiscsi_target_release(cs->cs_target);
1327         icl_conn_close(cs->cs_conn);
1328         icl_conn_free(cs->cs_conn);
1329         free(cs, M_CFISCSI);
1330         cv_signal(&softc->sessions_cv);
1331 }
1332
1333 static int
1334 cfiscsi_init(void)
1335 {
1336         struct cfiscsi_softc *softc;
1337
1338         softc = &cfiscsi_softc;
1339         bzero(softc, sizeof(*softc));
1340         mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1341
1342         cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1343 #ifdef ICL_KERNEL_PROXY
1344         cv_init(&softc->accept_cv, "cfiscsi_accept");
1345 #endif
1346         TAILQ_INIT(&softc->sessions);
1347         TAILQ_INIT(&softc->targets);
1348
1349         cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1350             sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1351             UMA_ALIGN_PTR, 0);
1352
1353         return (0);
1354 }
1355
1356 static int
1357 cfiscsi_shutdown(void)
1358 {
1359         struct cfiscsi_softc *softc = &cfiscsi_softc;
1360
1361         if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1362                 return (EBUSY);
1363
1364         uma_zdestroy(cfiscsi_data_wait_zone);
1365 #ifdef ICL_KERNEL_PROXY
1366         cv_destroy(&softc->accept_cv);
1367 #endif
1368         cv_destroy(&softc->sessions_cv);
1369         mtx_destroy(&softc->lock);
1370         return (0);
1371 }
1372
1373 #ifdef ICL_KERNEL_PROXY
1374 static void
1375 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1376 {
1377         struct cfiscsi_session *cs;
1378
1379         cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1380         if (cs == NULL) {
1381                 CFISCSI_WARN("failed to create session");
1382                 return;
1383         }
1384
1385         icl_conn_handoff_sock(cs->cs_conn, so);
1386         cs->cs_initiator_sa = sa;
1387         cs->cs_portal_id = portal_id;
1388         cs->cs_handoff_in_progress = false;
1389         cs->cs_waiting_for_ctld = true;
1390         cv_signal(&cfiscsi_softc.accept_cv);
1391
1392         CFISCSI_SESSION_LOCK(cs);
1393         /*
1394          * Wake up the maintenance thread if we got scheduled for termination
1395          * somewhere between cfiscsi_session_new() and icl_conn_handoff_sock().
1396          */
1397         if (cs->cs_terminating)
1398                 cfiscsi_session_terminate(cs);
1399         CFISCSI_SESSION_UNLOCK(cs);
1400 }
1401 #endif
1402
1403 static void
1404 cfiscsi_online(void *arg)
1405 {
1406         struct cfiscsi_softc *softc;
1407         struct cfiscsi_target *ct;
1408         int online;
1409
1410         ct = (struct cfiscsi_target *)arg;
1411         softc = ct->ct_softc;
1412
1413         mtx_lock(&softc->lock);
1414         if (ct->ct_online) {
1415                 mtx_unlock(&softc->lock);
1416                 return;
1417         }
1418         ct->ct_online = 1;
1419         online = softc->online++;
1420         mtx_unlock(&softc->lock);
1421         if (online > 0)
1422                 return;
1423
1424 #ifdef ICL_KERNEL_PROXY
1425         if (softc->listener != NULL)
1426                 icl_listen_free(softc->listener);
1427         softc->listener = icl_listen_new(cfiscsi_accept);
1428 #endif
1429 }
1430
1431 static void
1432 cfiscsi_offline(void *arg)
1433 {
1434         struct cfiscsi_softc *softc;
1435         struct cfiscsi_target *ct;
1436         struct cfiscsi_session *cs;
1437         int error, online;
1438
1439         ct = (struct cfiscsi_target *)arg;
1440         softc = ct->ct_softc;
1441
1442         mtx_lock(&softc->lock);
1443         if (!ct->ct_online) {
1444                 mtx_unlock(&softc->lock);
1445                 return;
1446         }
1447         ct->ct_online = 0;
1448         online = --softc->online;
1449
1450         do {
1451                 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1452                         if (cs->cs_target == ct)
1453                                 cfiscsi_session_terminate(cs);
1454                 }
1455                 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1456                         if (cs->cs_target == ct)
1457                                 break;
1458                 }
1459                 if (cs != NULL) {
1460                         error = cv_wait_sig(&softc->sessions_cv, &softc->lock);
1461                         if (error != 0) {
1462                                 CFISCSI_SESSION_DEBUG(cs,
1463                                     "cv_wait failed with error %d\n", error);
1464                                 break;
1465                         }
1466                 }
1467         } while (cs != NULL && ct->ct_online == 0);
1468         mtx_unlock(&softc->lock);
1469         if (online > 0)
1470                 return;
1471
1472 #ifdef ICL_KERNEL_PROXY
1473         icl_listen_free(softc->listener);
1474         softc->listener = NULL;
1475 #endif
1476 }
1477
1478 static int
1479 cfiscsi_info(void *arg, struct sbuf *sb)
1480 {
1481         struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1482         int retval;
1483
1484         retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1485             ct->ct_state);
1486         return (retval);
1487 }
1488
1489 static void
1490 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1491 {
1492         struct cfiscsi_softc *softc;
1493         struct cfiscsi_session *cs, *cs2;
1494         struct cfiscsi_target *ct;
1495         struct ctl_iscsi_handoff_params *cihp;
1496         int error;
1497
1498         cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1499         softc = &cfiscsi_softc;
1500
1501         CFISCSI_DEBUG("new connection from %s (%s) to %s",
1502             cihp->initiator_name, cihp->initiator_addr,
1503             cihp->target_name);
1504
1505         ct = cfiscsi_target_find(softc, cihp->target_name,
1506             cihp->portal_group_tag);
1507         if (ct == NULL) {
1508                 ci->status = CTL_ISCSI_ERROR;
1509                 snprintf(ci->error_str, sizeof(ci->error_str),
1510                     "%s: target not found", __func__);
1511                 return;
1512         }
1513
1514 #ifdef ICL_KERNEL_PROXY
1515         if (cihp->socket > 0 && cihp->connection_id > 0) {
1516                 snprintf(ci->error_str, sizeof(ci->error_str),
1517                     "both socket and connection_id set");
1518                 ci->status = CTL_ISCSI_ERROR;
1519                 cfiscsi_target_release(ct);
1520                 return;
1521         }
1522         if (cihp->socket == 0) {
1523                 mtx_lock(&cfiscsi_softc.lock);
1524                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1525                         if (cs->cs_id == cihp->connection_id)
1526                                 break;
1527                 }
1528                 if (cs == NULL) {
1529                         mtx_unlock(&cfiscsi_softc.lock);
1530                         snprintf(ci->error_str, sizeof(ci->error_str),
1531                             "connection not found");
1532                         ci->status = CTL_ISCSI_ERROR;
1533                         cfiscsi_target_release(ct);
1534                         return;
1535                 }
1536                 mtx_unlock(&cfiscsi_softc.lock);
1537         } else {
1538 #endif
1539                 cs = cfiscsi_session_new(softc, cihp->offload);
1540                 if (cs == NULL) {
1541                         ci->status = CTL_ISCSI_ERROR;
1542                         snprintf(ci->error_str, sizeof(ci->error_str),
1543                             "%s: cfiscsi_session_new failed", __func__);
1544                         cfiscsi_target_release(ct);
1545                         return;
1546                 }
1547 #ifdef ICL_KERNEL_PROXY
1548         }
1549 #endif
1550
1551         /*
1552          * First PDU of Full Feature phase has the same CmdSN as the last
1553          * PDU from the Login Phase received from the initiator.  Thus,
1554          * the -1 below.
1555          */
1556         cs->cs_cmdsn = cihp->cmdsn;
1557         cs->cs_statsn = cihp->statsn;
1558         cs->cs_max_recv_data_segment_length = cihp->max_recv_data_segment_length;
1559         cs->cs_max_send_data_segment_length = cihp->max_send_data_segment_length;
1560         cs->cs_max_burst_length = cihp->max_burst_length;
1561         cs->cs_first_burst_length = cihp->first_burst_length;
1562         cs->cs_immediate_data = !!cihp->immediate_data;
1563         if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1564                 cs->cs_conn->ic_header_crc32c = true;
1565         if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1566                 cs->cs_conn->ic_data_crc32c = true;
1567
1568         strlcpy(cs->cs_initiator_name,
1569             cihp->initiator_name, sizeof(cs->cs_initiator_name));
1570         strlcpy(cs->cs_initiator_addr,
1571             cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1572         strlcpy(cs->cs_initiator_alias,
1573             cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1574         memcpy(cs->cs_initiator_isid,
1575             cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1576         snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1577             "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1578             cihp->initiator_isid[0], cihp->initiator_isid[1],
1579             cihp->initiator_isid[2], cihp->initiator_isid[3],
1580             cihp->initiator_isid[4], cihp->initiator_isid[5]);
1581
1582         mtx_lock(&softc->lock);
1583         if (ct->ct_online == 0) {
1584                 mtx_unlock(&softc->lock);
1585                 cs->cs_handoff_in_progress = false;
1586                 cfiscsi_session_terminate(cs);
1587                 cfiscsi_target_release(ct);
1588                 ci->status = CTL_ISCSI_ERROR;
1589                 snprintf(ci->error_str, sizeof(ci->error_str),
1590                     "%s: port offline", __func__);
1591                 return;
1592         }
1593         cs->cs_target = ct;
1594         mtx_unlock(&softc->lock);
1595
1596 restart:
1597         if (!cs->cs_terminating) {
1598                 mtx_lock(&softc->lock);
1599                 TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1600                         if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1601                             cs->cs_target == cs2->cs_target &&
1602                             strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1603                                 if (strcmp(cs->cs_initiator_addr,
1604                                     cs2->cs_initiator_addr) != 0) {
1605                                         CFISCSI_SESSION_WARN(cs2,
1606                                             "session reinstatement from "
1607                                             "different address %s",
1608                                             cs->cs_initiator_addr);
1609                                 } else {
1610                                         CFISCSI_SESSION_DEBUG(cs2,
1611                                             "session reinstatement");
1612                                 }
1613                                 cfiscsi_session_terminate(cs2);
1614                                 mtx_unlock(&softc->lock);
1615                                 pause("cfiscsi_reinstate", 1);
1616                                 goto restart;
1617                         }
1618                 }
1619                 mtx_unlock(&softc->lock);
1620         }
1621
1622         /*
1623          * Register initiator with CTL.
1624          */
1625         cfiscsi_session_register_initiator(cs);
1626
1627 #ifdef ICL_KERNEL_PROXY
1628         if (cihp->socket > 0) {
1629 #endif
1630                 error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1631                 if (error != 0) {
1632                         cs->cs_handoff_in_progress = false;
1633                         cfiscsi_session_terminate(cs);
1634                         ci->status = CTL_ISCSI_ERROR;
1635                         snprintf(ci->error_str, sizeof(ci->error_str),
1636                             "%s: icl_conn_handoff failed with error %d",
1637                             __func__, error);
1638                         return;
1639                 }
1640 #ifdef ICL_KERNEL_PROXY
1641         }
1642 #endif
1643
1644 #ifdef ICL_KERNEL_PROXY
1645         cs->cs_login_phase = false;
1646
1647         /*
1648          * First PDU of the Full Feature phase has likely already arrived.
1649          * We have to pick it up and execute properly.
1650          */
1651         if (cs->cs_login_pdu != NULL) {
1652                 CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1653                 cfiscsi_pdu_handle(cs->cs_login_pdu);
1654                 cs->cs_login_pdu = NULL;
1655         }
1656 #endif
1657
1658         CFISCSI_SESSION_LOCK(cs);
1659         cs->cs_handoff_in_progress = false;
1660
1661         /*
1662          * Wake up the maintenance thread if we got scheduled for termination.
1663          */
1664         if (cs->cs_terminating)
1665                 cfiscsi_session_terminate(cs);
1666         CFISCSI_SESSION_UNLOCK(cs);
1667
1668         ci->status = CTL_ISCSI_OK;
1669 }
1670
1671 static void
1672 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1673 {
1674         struct ctl_iscsi_list_params *cilp;
1675         struct cfiscsi_session *cs;
1676         struct cfiscsi_softc *softc;
1677         struct sbuf *sb;
1678         int error;
1679
1680         cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1681         softc = &cfiscsi_softc;
1682
1683         sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1684         if (sb == NULL) {
1685                 ci->status = CTL_ISCSI_ERROR;
1686                 snprintf(ci->error_str, sizeof(ci->error_str),
1687                     "Unable to allocate %d bytes for iSCSI session list",
1688                     cilp->alloc_len);
1689                 return;
1690         }
1691
1692         sbuf_printf(sb, "<ctlislist>\n");
1693         mtx_lock(&softc->lock);
1694         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1695 #ifdef ICL_KERNEL_PROXY
1696                 if (cs->cs_target == NULL)
1697                         continue;
1698 #endif
1699                 error = sbuf_printf(sb, "<connection id=\"%d\">"
1700                     "<initiator>%s</initiator>"
1701                     "<initiator_addr>%s</initiator_addr>"
1702                     "<initiator_alias>%s</initiator_alias>"
1703                     "<target>%s</target>"
1704                     "<target_alias>%s</target_alias>"
1705                     "<target_portal_group_tag>%u</target_portal_group_tag>"
1706                     "<header_digest>%s</header_digest>"
1707                     "<data_digest>%s</data_digest>"
1708                     "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1709                     "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1710                     "<max_burst_length>%d</max_burst_length>"
1711                     "<first_burst_length>%d</first_burst_length>"
1712                     "<immediate_data>%d</immediate_data>"
1713                     "<iser>%d</iser>"
1714                     "<offload>%s</offload>"
1715                     "</connection>\n",
1716                     cs->cs_id,
1717                     cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1718                     cs->cs_target->ct_name, cs->cs_target->ct_alias,
1719                     cs->cs_target->ct_tag,
1720                     cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1721                     cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1722                     cs->cs_max_recv_data_segment_length,
1723                     cs->cs_max_send_data_segment_length,
1724                     cs->cs_max_burst_length,
1725                     cs->cs_first_burst_length,
1726                     cs->cs_immediate_data,
1727                     cs->cs_conn->ic_iser,
1728                     cs->cs_conn->ic_offload);
1729                 if (error != 0)
1730                         break;
1731         }
1732         mtx_unlock(&softc->lock);
1733         error = sbuf_printf(sb, "</ctlislist>\n");
1734         if (error != 0) {
1735                 sbuf_delete(sb);
1736                 ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1737                 snprintf(ci->error_str, sizeof(ci->error_str),
1738                     "Out of space, %d bytes is too small", cilp->alloc_len);
1739                 return;
1740         }
1741         sbuf_finish(sb);
1742
1743         error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1744         if (error != 0) {
1745                 sbuf_delete(sb);
1746                 snprintf(ci->error_str, sizeof(ci->error_str),
1747                     "copyout failed with error %d", error);
1748                 ci->status = CTL_ISCSI_ERROR;
1749                 return;
1750         }
1751         cilp->fill_len = sbuf_len(sb) + 1;
1752         ci->status = CTL_ISCSI_OK;
1753         sbuf_delete(sb);
1754 }
1755
1756 static void
1757 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1758 {
1759         struct icl_pdu *response;
1760         struct iscsi_bhs_asynchronous_message *bhsam;
1761         struct ctl_iscsi_logout_params *cilp;
1762         struct cfiscsi_session *cs;
1763         struct cfiscsi_softc *softc;
1764         int found = 0;
1765
1766         cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1767         softc = &cfiscsi_softc;
1768
1769         mtx_lock(&softc->lock);
1770         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1771                 if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1772                     strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1773                     strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1774                         continue;
1775
1776                 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1777                 if (response == NULL) {
1778                         ci->status = CTL_ISCSI_ERROR;
1779                         snprintf(ci->error_str, sizeof(ci->error_str),
1780                             "Unable to allocate memory");
1781                         mtx_unlock(&softc->lock);
1782                         return;
1783                 }
1784                 bhsam =
1785                     (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1786                 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1787                 bhsam->bhsam_flags = 0x80;
1788                 bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1789                 bhsam->bhsam_parameter3 = htons(10);
1790                 cfiscsi_pdu_queue(response);
1791                 found++;
1792         }
1793         mtx_unlock(&softc->lock);
1794
1795         if (found == 0) {
1796                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1797                 snprintf(ci->error_str, sizeof(ci->error_str),
1798                     "No matching connections found");
1799                 return;
1800         }
1801
1802         ci->status = CTL_ISCSI_OK;
1803 }
1804
1805 static void
1806 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1807 {
1808         struct icl_pdu *response;
1809         struct iscsi_bhs_asynchronous_message *bhsam;
1810         struct ctl_iscsi_terminate_params *citp;
1811         struct cfiscsi_session *cs;
1812         struct cfiscsi_softc *softc;
1813         int found = 0;
1814
1815         citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1816         softc = &cfiscsi_softc;
1817
1818         mtx_lock(&softc->lock);
1819         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1820                 if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1821                     strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1822                     strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1823                         continue;
1824
1825                 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1826                 if (response == NULL) {
1827                         /*
1828                          * Oh well.  Just terminate the connection.
1829                          */
1830                 } else {
1831                         bhsam = (struct iscsi_bhs_asynchronous_message *)
1832                             response->ip_bhs;
1833                         bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1834                         bhsam->bhsam_flags = 0x80;
1835                         bhsam->bhsam_0xffffffff = 0xffffffff;
1836                         bhsam->bhsam_async_event =
1837                             BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1838                         cfiscsi_pdu_queue(response);
1839                 }
1840                 cfiscsi_session_terminate(cs);
1841                 found++;
1842         }
1843         mtx_unlock(&softc->lock);
1844
1845         if (found == 0) {
1846                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1847                 snprintf(ci->error_str, sizeof(ci->error_str),
1848                     "No matching connections found");
1849                 return;
1850         }
1851
1852         ci->status = CTL_ISCSI_OK;
1853 }
1854
1855 static void
1856 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1857 {
1858         struct ctl_iscsi_limits_params *cilp;
1859         struct icl_drv_limits idl;
1860         int error;
1861
1862         cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1863
1864         error = icl_limits(cilp->offload, false, &idl);
1865         if (error != 0) {
1866                 ci->status = CTL_ISCSI_ERROR;
1867                 snprintf(ci->error_str, sizeof(ci->error_str),
1868                         "%s: icl_limits failed with error %d",
1869                         __func__, error);
1870                 return;
1871         }
1872
1873         cilp->max_recv_data_segment_length =
1874             idl.idl_max_recv_data_segment_length;
1875         cilp->max_send_data_segment_length =
1876             idl.idl_max_send_data_segment_length;
1877         cilp->max_burst_length = idl.idl_max_burst_length;
1878         cilp->first_burst_length = idl.idl_first_burst_length;
1879
1880         ci->status = CTL_ISCSI_OK;
1881 }
1882
1883 #ifdef ICL_KERNEL_PROXY
1884 static void
1885 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1886 {
1887         struct ctl_iscsi_listen_params *cilp;
1888         struct sockaddr *sa;
1889         int error;
1890
1891         cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1892
1893         if (cfiscsi_softc.listener == NULL) {
1894                 CFISCSI_DEBUG("no listener");
1895                 snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1896                 ci->status = CTL_ISCSI_ERROR;
1897                 return;
1898         }
1899
1900         error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1901         if (error != 0) {
1902                 CFISCSI_DEBUG("getsockaddr, error %d", error);
1903                 snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1904                 ci->status = CTL_ISCSI_ERROR;
1905                 return;
1906         }
1907
1908         error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1909             cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1910         if (error != 0) {
1911                 free(sa, M_SONAME);
1912                 CFISCSI_DEBUG("icl_listen_add, error %d", error);
1913                 snprintf(ci->error_str, sizeof(ci->error_str),
1914                     "icl_listen_add failed, error %d", error);
1915                 ci->status = CTL_ISCSI_ERROR;
1916                 return;
1917         }
1918
1919         ci->status = CTL_ISCSI_OK;
1920 }
1921
1922 static void
1923 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1924 {
1925         struct ctl_iscsi_accept_params *ciap;
1926         struct cfiscsi_session *cs;
1927         int error;
1928
1929         ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1930
1931         mtx_lock(&cfiscsi_softc.lock);
1932         for (;;) {
1933                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1934                         if (cs->cs_waiting_for_ctld)
1935                                 break;
1936                 }
1937                 if (cs != NULL)
1938                         break;
1939                 error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1940                 if (error != 0) {
1941                         mtx_unlock(&cfiscsi_softc.lock);
1942                         snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1943                         ci->status = CTL_ISCSI_ERROR;
1944                         return;
1945                 }
1946         }
1947         mtx_unlock(&cfiscsi_softc.lock);
1948
1949         cs->cs_waiting_for_ctld = false;
1950         cs->cs_login_phase = true;
1951
1952         ciap->connection_id = cs->cs_id;
1953         ciap->portal_id = cs->cs_portal_id;
1954         ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1955         error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1956             cs->cs_initiator_sa->sa_len);
1957         if (error != 0) {
1958                 snprintf(ci->error_str, sizeof(ci->error_str),
1959                     "copyout failed with error %d", error);
1960                 ci->status = CTL_ISCSI_ERROR;
1961                 return;
1962         }
1963
1964         ci->status = CTL_ISCSI_OK;
1965 }
1966
1967 static void
1968 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1969 {
1970         struct ctl_iscsi_send_params *cisp;
1971         struct cfiscsi_session *cs;
1972         struct icl_pdu *ip;
1973         size_t datalen;
1974         void *data;
1975         int error;
1976
1977         cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1978
1979         mtx_lock(&cfiscsi_softc.lock);
1980         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1981                 if (cs->cs_id == cisp->connection_id)
1982                         break;
1983         }
1984         if (cs == NULL) {
1985                 mtx_unlock(&cfiscsi_softc.lock);
1986                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1987                 ci->status = CTL_ISCSI_ERROR;
1988                 return;
1989         }
1990         mtx_unlock(&cfiscsi_softc.lock);
1991
1992 #if 0
1993         if (cs->cs_login_phase == false)
1994                 return (EBUSY);
1995 #endif
1996
1997         if (cs->cs_terminating) {
1998                 snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1999                 ci->status = CTL_ISCSI_ERROR;
2000                 return;
2001         }
2002
2003         datalen = cisp->data_segment_len;
2004         /*
2005          * XXX
2006          */
2007         //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
2008         if (datalen > 65535) {
2009                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
2010                 ci->status = CTL_ISCSI_ERROR;
2011                 return;
2012         }
2013         if (datalen > 0) {
2014                 data = malloc(datalen, M_CFISCSI, M_WAITOK);
2015                 error = copyin(cisp->data_segment, data, datalen);
2016                 if (error != 0) {
2017                         free(data, M_CFISCSI);
2018                         snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
2019                         ci->status = CTL_ISCSI_ERROR;
2020                         return;
2021                 }
2022         }
2023
2024         ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
2025         memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
2026         if (datalen > 0) {
2027                 icl_pdu_append_data(ip, data, datalen, M_WAITOK);
2028                 free(data, M_CFISCSI);
2029         }
2030         CFISCSI_SESSION_LOCK(cs);
2031         icl_pdu_queue(ip);
2032         CFISCSI_SESSION_UNLOCK(cs);
2033         ci->status = CTL_ISCSI_OK;
2034 }
2035
2036 static void
2037 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
2038 {
2039         struct ctl_iscsi_receive_params *cirp;
2040         struct cfiscsi_session *cs;
2041         struct icl_pdu *ip;
2042         void *data;
2043         int error;
2044
2045         cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2046
2047         mtx_lock(&cfiscsi_softc.lock);
2048         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2049                 if (cs->cs_id == cirp->connection_id)
2050                         break;
2051         }
2052         if (cs == NULL) {
2053                 mtx_unlock(&cfiscsi_softc.lock);
2054                 snprintf(ci->error_str, sizeof(ci->error_str),
2055                     "connection not found");
2056                 ci->status = CTL_ISCSI_ERROR;
2057                 return;
2058         }
2059         mtx_unlock(&cfiscsi_softc.lock);
2060
2061 #if 0
2062         if (is->is_login_phase == false)
2063                 return (EBUSY);
2064 #endif
2065
2066         CFISCSI_SESSION_LOCK(cs);
2067         while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2068                 error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2069                 if (error != 0) {
2070                         CFISCSI_SESSION_UNLOCK(cs);
2071                         snprintf(ci->error_str, sizeof(ci->error_str),
2072                             "interrupted by signal");
2073                         ci->status = CTL_ISCSI_ERROR;
2074                         return;
2075                 }
2076         }
2077
2078         if (cs->cs_terminating) {
2079                 CFISCSI_SESSION_UNLOCK(cs);
2080                 snprintf(ci->error_str, sizeof(ci->error_str),
2081                     "connection terminating");
2082                 ci->status = CTL_ISCSI_ERROR;
2083                 return;
2084         }
2085         ip = cs->cs_login_pdu;
2086         cs->cs_login_pdu = NULL;
2087         CFISCSI_SESSION_UNLOCK(cs);
2088
2089         if (ip->ip_data_len > cirp->data_segment_len) {
2090                 icl_pdu_free(ip);
2091                 snprintf(ci->error_str, sizeof(ci->error_str),
2092                     "data segment too big");
2093                 ci->status = CTL_ISCSI_ERROR;
2094                 return;
2095         }
2096
2097         copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2098         if (ip->ip_data_len > 0) {
2099                 data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2100                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2101                 copyout(data, cirp->data_segment, ip->ip_data_len);
2102                 free(data, M_CFISCSI);
2103         }
2104
2105         icl_pdu_free(ip);
2106         ci->status = CTL_ISCSI_OK;
2107 }
2108
2109 #endif /* !ICL_KERNEL_PROXY */
2110
2111 static void
2112 cfiscsi_ioctl_port_create(struct ctl_req *req)
2113 {
2114         struct cfiscsi_target *ct;
2115         struct ctl_port *port;
2116         const char *target, *alias, *val;
2117         struct scsi_vpd_id_descriptor *desc;
2118         int retval, len, idlen;
2119         uint16_t tag;
2120
2121         target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2122         alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);
2123         val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2124             NULL);
2125
2126
2127         if (target == NULL || val == NULL) {
2128                 req->status = CTL_LUN_ERROR;
2129                 snprintf(req->error_str, sizeof(req->error_str),
2130                     "Missing required argument");
2131                 return;
2132         }
2133
2134         tag = strtoul(val, NULL, 0);
2135         ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2136         if (ct == NULL) {
2137                 req->status = CTL_LUN_ERROR;
2138                 snprintf(req->error_str, sizeof(req->error_str),
2139                     "failed to create target \"%s\"", target);
2140                 return;
2141         }
2142         if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2143                 req->status = CTL_LUN_ERROR;
2144                 snprintf(req->error_str, sizeof(req->error_str),
2145                     "target \"%s\" for portal group tag %u already exists",
2146                     target, tag);
2147                 cfiscsi_target_release(ct);
2148                 return;
2149         }
2150         port = &ct->ct_port;
2151         // WAT
2152         if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2153                 goto done;
2154
2155         port->frontend = &cfiscsi_frontend;
2156         port->port_type = CTL_PORT_ISCSI;
2157         /* XXX KDM what should the real number be here? */
2158         port->num_requested_ctl_io = 4096;
2159         port->port_name = "iscsi";
2160         port->physical_port = (int)tag;
2161         port->virtual_port = ct->ct_target_id;
2162         port->port_online = cfiscsi_online;
2163         port->port_offline = cfiscsi_offline;
2164         port->port_info = cfiscsi_info;
2165         port->onoff_arg = ct;
2166         port->fe_datamove = cfiscsi_datamove;
2167         port->fe_done = cfiscsi_done;
2168         port->targ_port = -1;
2169         port->options = nvlist_clone(req->args_nvl);
2170
2171         /* Generate Port ID. */
2172         idlen = strlen(target) + strlen(",t,0x0001") + 1;
2173         idlen = roundup2(idlen, 4);
2174         len = sizeof(struct scsi_vpd_device_id) + idlen;
2175         port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2176             M_CTL, M_WAITOK | M_ZERO);
2177         port->port_devid->len = len;
2178         desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2179         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2180         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2181             SVPD_ID_TYPE_SCSI_NAME;
2182         desc->length = idlen;
2183         snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2184
2185         /* Generate Target ID. */
2186         idlen = strlen(target) + 1;
2187         idlen = roundup2(idlen, 4);
2188         len = sizeof(struct scsi_vpd_device_id) + idlen;
2189         port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2190             M_CTL, M_WAITOK | M_ZERO);
2191         port->target_devid->len = len;
2192         desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2193         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2194         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2195             SVPD_ID_TYPE_SCSI_NAME;
2196         desc->length = idlen;
2197         strlcpy(desc->identifier, target, idlen);
2198
2199         retval = ctl_port_register(port);
2200         if (retval != 0) {
2201                 free(port->port_devid, M_CFISCSI);
2202                 free(port->target_devid, M_CFISCSI);
2203                 cfiscsi_target_release(ct);
2204                 req->status = CTL_LUN_ERROR;
2205                 snprintf(req->error_str, sizeof(req->error_str),
2206                     "ctl_port_register() failed with error %d", retval);
2207                 return;
2208         }
2209 done:
2210         ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2211         req->status = CTL_LUN_OK;
2212         req->result_nvl = nvlist_create(0);
2213         nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
2214 }
2215
2216 static void
2217 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2218 {
2219         struct cfiscsi_target *ct;
2220         const char *target, *val;
2221         uint16_t tag;
2222
2223         target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2224         val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2225             NULL);
2226
2227         if (target == NULL || val == NULL) {
2228                 req->status = CTL_LUN_ERROR;
2229                 snprintf(req->error_str, sizeof(req->error_str),
2230                     "Missing required argument");
2231                 return;
2232         }
2233
2234         tag = strtoul(val, NULL, 0);
2235         ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2236         if (ct == NULL) {
2237                 req->status = CTL_LUN_ERROR;
2238                 snprintf(req->error_str, sizeof(req->error_str),
2239                     "can't find target \"%s\"", target);
2240                 return;
2241         }
2242
2243         ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2244         ctl_port_offline(&ct->ct_port);
2245         cfiscsi_target_release(ct);
2246         cfiscsi_target_release(ct);
2247         req->status = CTL_LUN_OK;
2248 }
2249
2250 static int
2251 cfiscsi_ioctl(struct cdev *dev,
2252     u_long cmd, caddr_t addr, int flag, struct thread *td)
2253 {
2254         struct ctl_iscsi *ci;
2255         struct ctl_req *req;
2256
2257         if (cmd == CTL_PORT_REQ) {
2258                 req = (struct ctl_req *)addr;
2259                 switch (req->reqtype) {
2260                 case CTL_REQ_CREATE:
2261                         cfiscsi_ioctl_port_create(req);
2262                         break;
2263                 case CTL_REQ_REMOVE:
2264                         cfiscsi_ioctl_port_remove(req);
2265                         break;
2266                 default:
2267                         req->status = CTL_LUN_ERROR;
2268                         snprintf(req->error_str, sizeof(req->error_str),
2269                             "Unsupported request type %d", req->reqtype);
2270                 }
2271                 return (0);
2272         }
2273
2274         if (cmd != CTL_ISCSI)
2275                 return (ENOTTY);
2276
2277         ci = (struct ctl_iscsi *)addr;
2278         switch (ci->type) {
2279         case CTL_ISCSI_HANDOFF:
2280                 cfiscsi_ioctl_handoff(ci);
2281                 break;
2282         case CTL_ISCSI_LIST:
2283                 cfiscsi_ioctl_list(ci);
2284                 break;
2285         case CTL_ISCSI_LOGOUT:
2286                 cfiscsi_ioctl_logout(ci);
2287                 break;
2288         case CTL_ISCSI_TERMINATE:
2289                 cfiscsi_ioctl_terminate(ci);
2290                 break;
2291         case CTL_ISCSI_LIMITS:
2292                 cfiscsi_ioctl_limits(ci);
2293                 break;
2294 #ifdef ICL_KERNEL_PROXY
2295         case CTL_ISCSI_LISTEN:
2296                 cfiscsi_ioctl_listen(ci);
2297                 break;
2298         case CTL_ISCSI_ACCEPT:
2299                 cfiscsi_ioctl_accept(ci);
2300                 break;
2301         case CTL_ISCSI_SEND:
2302                 cfiscsi_ioctl_send(ci);
2303                 break;
2304         case CTL_ISCSI_RECEIVE:
2305                 cfiscsi_ioctl_receive(ci);
2306                 break;
2307 #else
2308         case CTL_ISCSI_LISTEN:
2309         case CTL_ISCSI_ACCEPT:
2310         case CTL_ISCSI_SEND:
2311         case CTL_ISCSI_RECEIVE:
2312                 ci->status = CTL_ISCSI_ERROR;
2313                 snprintf(ci->error_str, sizeof(ci->error_str),
2314                     "%s: CTL compiled without ICL_KERNEL_PROXY",
2315                     __func__);
2316                 break;
2317 #endif /* !ICL_KERNEL_PROXY */
2318         default:
2319                 ci->status = CTL_ISCSI_ERROR;
2320                 snprintf(ci->error_str, sizeof(ci->error_str),
2321                     "%s: invalid iSCSI request type %d", __func__, ci->type);
2322                 break;
2323         }
2324
2325         return (0);
2326 }
2327
2328 static void
2329 cfiscsi_target_hold(struct cfiscsi_target *ct)
2330 {
2331
2332         refcount_acquire(&ct->ct_refcount);
2333 }
2334
2335 static void
2336 cfiscsi_target_release(struct cfiscsi_target *ct)
2337 {
2338         struct cfiscsi_softc *softc;
2339
2340         softc = ct->ct_softc;
2341         mtx_lock(&softc->lock);
2342         if (refcount_release(&ct->ct_refcount)) {
2343                 TAILQ_REMOVE(&softc->targets, ct, ct_next);
2344                 mtx_unlock(&softc->lock);
2345                 if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2346                         ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2347                         if (ctl_port_deregister(&ct->ct_port) != 0)
2348                                 printf("%s: ctl_port_deregister() failed\n",
2349                                     __func__);
2350                 }
2351                 free(ct, M_CFISCSI);
2352
2353                 return;
2354         }
2355         mtx_unlock(&softc->lock);
2356 }
2357
2358 static struct cfiscsi_target *
2359 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2360 {
2361         struct cfiscsi_target *ct;
2362
2363         mtx_lock(&softc->lock);
2364         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2365                 if (ct->ct_tag != tag ||
2366                     strcmp(name, ct->ct_name) != 0 ||
2367                     ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2368                         continue;
2369                 cfiscsi_target_hold(ct);
2370                 mtx_unlock(&softc->lock);
2371                 return (ct);
2372         }
2373         mtx_unlock(&softc->lock);
2374
2375         return (NULL);
2376 }
2377
2378 static struct cfiscsi_target *
2379 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2380     const char *alias, uint16_t tag)
2381 {
2382         struct cfiscsi_target *ct, *newct;
2383
2384         if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2385                 return (NULL);
2386
2387         newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2388
2389         mtx_lock(&softc->lock);
2390         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2391                 if (ct->ct_tag != tag ||
2392                     strcmp(name, ct->ct_name) != 0 ||
2393                     ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2394                         continue;
2395                 cfiscsi_target_hold(ct);
2396                 mtx_unlock(&softc->lock);
2397                 free(newct, M_CFISCSI);
2398                 return (ct);
2399         }
2400
2401         strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2402         if (alias != NULL)
2403                 strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2404         newct->ct_tag = tag;
2405         refcount_init(&newct->ct_refcount, 1);
2406         newct->ct_softc = softc;
2407         if (TAILQ_EMPTY(&softc->targets))
2408                 softc->last_target_id = 0;
2409         newct->ct_target_id = ++softc->last_target_id;
2410         TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2411         mtx_unlock(&softc->lock);
2412
2413         return (newct);
2414 }
2415
2416 static void
2417 cfiscsi_datamove_in(union ctl_io *io)
2418 {
2419         struct cfiscsi_session *cs;
2420         struct icl_pdu *request, *response;
2421         const struct iscsi_bhs_scsi_command *bhssc;
2422         struct iscsi_bhs_data_in *bhsdi;
2423         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2424         size_t len, expected_len, sg_len, buffer_offset;
2425         const char *sg_addr;
2426         int ctl_sg_count, error, i;
2427
2428         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2429         cs = PDU_SESSION(request);
2430
2431         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2432         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2433             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2434             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2435
2436         if (io->scsiio.kern_sg_entries > 0) {
2437                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2438                 ctl_sg_count = io->scsiio.kern_sg_entries;
2439         } else {
2440                 ctl_sglist = &ctl_sg_entry;
2441                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2442                 ctl_sglist->len = io->scsiio.kern_data_len;
2443                 ctl_sg_count = 1;
2444         }
2445
2446         /*
2447          * This is the total amount of data to be transferred within the current
2448          * SCSI command.  We need to record it so that we can properly report
2449          * underflow/underflow.
2450          */
2451         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2452
2453         /*
2454          * This is the offset within the current SCSI command; for the first
2455          * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2456          * it will be the sum of lengths of previous ones.
2457          */
2458         buffer_offset = io->scsiio.kern_rel_offset;
2459
2460         /*
2461          * This is the transfer length expected by the initiator.  In theory,
2462          * it could be different from the correct amount of data from the SCSI
2463          * point of view, even if that doesn't make any sense.
2464          */
2465         expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2466 #if 0
2467         if (expected_len != io->scsiio.kern_total_len) {
2468                 CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, "
2469                     "actual length %zd", expected_len,
2470                     (size_t)io->scsiio.kern_total_len);
2471         }
2472 #endif
2473
2474         if (buffer_offset >= expected_len) {
2475 #if 0
2476                 CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2477                     "already sent the expected len", buffer_offset);
2478 #endif
2479                 io->scsiio.be_move_done(io);
2480                 return;
2481         }
2482
2483         i = 0;
2484         sg_addr = NULL;
2485         sg_len = 0;
2486         response = NULL;
2487         bhsdi = NULL;
2488         for (;;) {
2489                 if (response == NULL) {
2490                         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2491                         if (response == NULL) {
2492                                 CFISCSI_SESSION_WARN(cs, "failed to "
2493                                     "allocate memory; dropping connection");
2494                                 ctl_set_busy(&io->scsiio);
2495                                 io->scsiio.be_move_done(io);
2496                                 cfiscsi_session_terminate(cs);
2497                                 return;
2498                         }
2499                         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2500                         bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2501                         bhsdi->bhsdi_initiator_task_tag =
2502                             bhssc->bhssc_initiator_task_tag;
2503                         bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2504                         bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2505                         PDU_EXPDATASN(request)++;
2506                         bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2507                 }
2508
2509                 KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2510                 if (sg_len == 0) {
2511                         sg_addr = ctl_sglist[i].addr;
2512                         sg_len = ctl_sglist[i].len;
2513                         KASSERT(sg_len > 0, ("sg_len <= 0"));
2514                 }
2515
2516                 len = sg_len;
2517
2518                 /*
2519                  * Truncate to maximum data segment length.
2520                  */
2521                 KASSERT(response->ip_data_len < cs->cs_max_send_data_segment_length,
2522                     ("ip_data_len %zd >= max_send_data_segment_length %d",
2523                     response->ip_data_len, cs->cs_max_send_data_segment_length));
2524                 if (response->ip_data_len + len >
2525                     cs->cs_max_send_data_segment_length) {
2526                         len = cs->cs_max_send_data_segment_length -
2527                             response->ip_data_len;
2528                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2529                             len, sg_len));
2530                 }
2531
2532                 /*
2533                  * Truncate to expected data transfer length.
2534                  */
2535                 KASSERT(buffer_offset + response->ip_data_len < expected_len,
2536                     ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2537                     buffer_offset, response->ip_data_len, expected_len));
2538                 if (buffer_offset + response->ip_data_len + len > expected_len) {
2539                         CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2540                             "to expected data transfer length %zd",
2541                             buffer_offset + response->ip_data_len + len, expected_len);
2542                         len = expected_len - (buffer_offset + response->ip_data_len);
2543                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2544                             len, sg_len));
2545                 }
2546
2547                 error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT);
2548                 if (error != 0) {
2549                         CFISCSI_SESSION_WARN(cs, "failed to "
2550                             "allocate memory; dropping connection");
2551                         icl_pdu_free(response);
2552                         ctl_set_busy(&io->scsiio);
2553                         io->scsiio.be_move_done(io);
2554                         cfiscsi_session_terminate(cs);
2555                         return;
2556                 }
2557                 sg_addr += len;
2558                 sg_len -= len;
2559                 io->scsiio.kern_data_resid -= len;
2560
2561                 KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2562                     ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2563                     buffer_offset, response->ip_data_len, expected_len));
2564                 if (buffer_offset + response->ip_data_len == expected_len) {
2565                         /*
2566                          * Already have the amount of data the initiator wanted.
2567                          */
2568                         break;
2569                 }
2570
2571                 if (sg_len == 0) {
2572                         /*
2573                          * End of scatter-gather segment;
2574                          * proceed to the next one...
2575                          */
2576                         if (i == ctl_sg_count - 1) {
2577                                 /*
2578                                  * ... unless this was the last one.
2579                                  */
2580                                 break;
2581                         }
2582                         i++;
2583                 }
2584
2585                 if (response->ip_data_len == cs->cs_max_send_data_segment_length) {
2586                         /*
2587                          * Can't stuff more data into the current PDU;
2588                          * queue it.  Note that's not enough to check
2589                          * for kern_data_resid == 0 instead; there
2590                          * may be several Data-In PDUs for the final
2591                          * call to cfiscsi_datamove(), and we want
2592                          * to set the F flag only on the last of them.
2593                          */
2594                         buffer_offset += response->ip_data_len;
2595                         if (buffer_offset == io->scsiio.kern_total_len ||
2596                             buffer_offset == expected_len) {
2597                                 buffer_offset -= response->ip_data_len;
2598                                 break;
2599                         }
2600                         cfiscsi_pdu_queue(response);
2601                         response = NULL;
2602                         bhsdi = NULL;
2603                 }
2604         }
2605         if (response != NULL) {
2606                 buffer_offset += response->ip_data_len;
2607                 if (buffer_offset == io->scsiio.kern_total_len ||
2608                     buffer_offset == expected_len) {
2609                         bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2610                         if (io->io_hdr.status == CTL_SUCCESS) {
2611                                 bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2612                                 if (PDU_TOTAL_TRANSFER_LEN(request) <
2613                                     ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2614                                         bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2615                                         bhsdi->bhsdi_residual_count =
2616                                             htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2617                                             PDU_TOTAL_TRANSFER_LEN(request));
2618                                 } else if (PDU_TOTAL_TRANSFER_LEN(request) >
2619                                     ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2620                                         bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2621                                         bhsdi->bhsdi_residual_count =
2622                                             htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2623                                             ntohl(bhssc->bhssc_expected_data_transfer_length));
2624                                 }
2625                                 bhsdi->bhsdi_status = io->scsiio.scsi_status;
2626                                 io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2627                         }
2628                 }
2629                 KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2630                 cfiscsi_pdu_queue(response);
2631         }
2632
2633         io->scsiio.be_move_done(io);
2634 }
2635
2636 static void
2637 cfiscsi_datamove_out(union ctl_io *io)
2638 {
2639         struct cfiscsi_session *cs;
2640         struct icl_pdu *request, *response;
2641         const struct iscsi_bhs_scsi_command *bhssc;
2642         struct iscsi_bhs_r2t *bhsr2t;
2643         struct cfiscsi_data_wait *cdw;
2644         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2645         uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2646         uint32_t target_transfer_tag;
2647         bool done;
2648
2649         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2650         cs = PDU_SESSION(request);
2651
2652         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2653         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2654             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2655             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2656
2657         /*
2658          * We need to record it so that we can properly report
2659          * underflow/underflow.
2660          */
2661         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2662
2663         /*
2664          * Complete write underflow.  Not a single byte to read.  Return.
2665          */
2666         expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2667         if (io->scsiio.kern_rel_offset >= expected_len) {
2668                 io->scsiio.be_move_done(io);
2669                 return;
2670         }
2671         datamove_len = MIN(io->scsiio.kern_data_len,
2672             expected_len - io->scsiio.kern_rel_offset);
2673
2674         target_transfer_tag =
2675             atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2676         cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2677             &target_transfer_tag);
2678         if (cdw == NULL) {
2679                 CFISCSI_SESSION_WARN(cs, "failed to "
2680                     "allocate memory; dropping connection");
2681                 ctl_set_busy(&io->scsiio);
2682                 io->scsiio.be_move_done(io);
2683                 cfiscsi_session_terminate(cs);
2684                 return;
2685         }
2686 #if 0
2687         CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2688             "task tag 0x%x, target transfer tag 0x%x",
2689             bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2690 #endif
2691
2692         cdw->cdw_ctl_io = io;
2693         cdw->cdw_target_transfer_tag = target_transfer_tag;
2694         cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2695         cdw->cdw_r2t_end = datamove_len;
2696         cdw->cdw_datasn = 0;
2697
2698         /* Set initial data pointer for the CDW respecting ext_data_filled. */
2699         if (io->scsiio.kern_sg_entries > 0) {
2700                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2701         } else {
2702                 ctl_sglist = &ctl_sg_entry;
2703                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2704                 ctl_sglist->len = datamove_len;
2705         }
2706         cdw->cdw_sg_index = 0;
2707         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2708         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2709         r2t_off = io->scsiio.ext_data_filled;
2710         while (r2t_off > 0) {
2711                 if (r2t_off >= cdw->cdw_sg_len) {
2712                         r2t_off -= cdw->cdw_sg_len;
2713                         cdw->cdw_sg_index++;
2714                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2715                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2716                         continue;
2717                 }
2718                 cdw->cdw_sg_addr += r2t_off;
2719                 cdw->cdw_sg_len -= r2t_off;
2720                 r2t_off = 0;
2721         }
2722
2723         if (cs->cs_immediate_data &&
2724             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2725             icl_pdu_data_segment_length(request)) {
2726                 done = cfiscsi_handle_data_segment(request, cdw);
2727                 if (done) {
2728                         cfiscsi_data_wait_free(cs, cdw);
2729                         io->scsiio.be_move_done(io);
2730                         return;
2731                 }
2732         }
2733
2734         r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2735         r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2736             cs->cs_max_burst_length);
2737         cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2738
2739         CFISCSI_SESSION_LOCK(cs);
2740         TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2741         CFISCSI_SESSION_UNLOCK(cs);
2742
2743         /*
2744          * XXX: We should limit the number of outstanding R2T PDUs
2745          *      per task to MaxOutstandingR2T.
2746          */
2747         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2748         if (response == NULL) {
2749                 CFISCSI_SESSION_WARN(cs, "failed to "
2750                     "allocate memory; dropping connection");
2751                 ctl_set_busy(&io->scsiio);
2752                 io->scsiio.be_move_done(io);
2753                 cfiscsi_session_terminate(cs);
2754                 return;
2755         }
2756         io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2757         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2758         bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2759         bhsr2t->bhsr2t_flags = 0x80;
2760         bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2761         bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2762         bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2763         /*
2764          * XXX: Here we assume that cfiscsi_datamove() won't ever
2765          *      be running concurrently on several CPUs for a given
2766          *      command.
2767          */
2768         bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2769         PDU_R2TSN(request)++;
2770         /*
2771          * This is the offset within the current SCSI command;
2772          * i.e. for the first call of datamove(), it will be 0,
2773          * and for subsequent ones it will be the sum of lengths
2774          * of previous ones.
2775          *
2776          * The ext_data_filled is to account for unsolicited
2777          * (immediate) data that might have already arrived.
2778          */
2779         bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2780         /*
2781          * This is the total length (sum of S/G lengths) this call
2782          * to cfiscsi_datamove() is supposed to handle, limited by
2783          * MaxBurstLength.
2784          */
2785         bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2786         cfiscsi_pdu_queue(response);
2787 }
2788
2789 static void
2790 cfiscsi_datamove(union ctl_io *io)
2791 {
2792
2793         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2794                 cfiscsi_datamove_in(io);
2795         else {
2796                 /* We hadn't received anything during this datamove yet. */
2797                 io->scsiio.ext_data_filled = 0;
2798                 cfiscsi_datamove_out(io);
2799         }
2800 }
2801
2802 static void
2803 cfiscsi_scsi_command_done(union ctl_io *io)
2804 {
2805         struct icl_pdu *request, *response;
2806         struct iscsi_bhs_scsi_command *bhssc;
2807         struct iscsi_bhs_scsi_response *bhssr;
2808 #ifdef DIAGNOSTIC
2809         struct cfiscsi_data_wait *cdw;
2810 #endif
2811         struct cfiscsi_session *cs;
2812         uint16_t sense_length;
2813
2814         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2815         cs = PDU_SESSION(request);
2816         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2817         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2818             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2819             ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2820
2821         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2822         //    bhssc->bhssc_initiator_task_tag);
2823
2824 #ifdef DIAGNOSTIC
2825         CFISCSI_SESSION_LOCK(cs);
2826         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2827                 KASSERT(bhssc->bhssc_initiator_task_tag !=
2828                     cdw->cdw_initiator_task_tag, ("dangling cdw"));
2829         CFISCSI_SESSION_UNLOCK(cs);
2830 #endif
2831
2832         /*
2833          * Do not return status for aborted commands.
2834          * There are exceptions, but none supported by CTL yet.
2835          */
2836         if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2837              (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2838             (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2839                 ctl_free_io(io);
2840                 icl_pdu_free(request);
2841                 return;
2842         }
2843
2844         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2845         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2846         bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2847         bhssr->bhssr_flags = 0x80;
2848         /*
2849          * XXX: We don't deal with bidirectional under/overflows;
2850          *      does anything actually support those?
2851          */
2852         if (PDU_TOTAL_TRANSFER_LEN(request) <
2853             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2854                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2855                 bhssr->bhssr_residual_count =
2856                     htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2857                     PDU_TOTAL_TRANSFER_LEN(request));
2858                 //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2859                 //    ntohl(bhssr->bhssr_residual_count));
2860         } else if (PDU_TOTAL_TRANSFER_LEN(request) >
2861             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2862                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2863                 bhssr->bhssr_residual_count =
2864                     htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2865                     ntohl(bhssc->bhssc_expected_data_transfer_length));
2866                 //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2867                 //    ntohl(bhssr->bhssr_residual_count));
2868         }
2869         bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2870         bhssr->bhssr_status = io->scsiio.scsi_status;
2871         bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2872         bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2873
2874         if (io->scsiio.sense_len > 0) {
2875 #if 0
2876                 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2877                     io->scsiio.sense_len);
2878 #endif
2879                 sense_length = htons(io->scsiio.sense_len);
2880                 icl_pdu_append_data(response,
2881                     &sense_length, sizeof(sense_length), M_WAITOK);
2882                 icl_pdu_append_data(response,
2883                     &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2884         }
2885
2886         ctl_free_io(io);
2887         icl_pdu_free(request);
2888         cfiscsi_pdu_queue(response);
2889 }
2890
2891 static void
2892 cfiscsi_task_management_done(union ctl_io *io)
2893 {
2894         struct icl_pdu *request, *response;
2895         struct iscsi_bhs_task_management_request *bhstmr;
2896         struct iscsi_bhs_task_management_response *bhstmr2;
2897         struct cfiscsi_data_wait *cdw, *tmpcdw;
2898         struct cfiscsi_session *cs, *tcs;
2899         struct cfiscsi_softc *softc;
2900         int cold_reset = 0;
2901
2902         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2903         cs = PDU_SESSION(request);
2904         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2905         KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2906             ISCSI_BHS_OPCODE_TASK_REQUEST,
2907             ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2908
2909 #if 0
2910         CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2911             bhstmr->bhstmr_initiator_task_tag,
2912             bhstmr->bhstmr_referenced_task_tag);
2913 #endif
2914
2915         if ((bhstmr->bhstmr_function & ~0x80) ==
2916             BHSTMR_FUNCTION_ABORT_TASK) {
2917                 /*
2918                  * Make sure we no longer wait for Data-Out for this command.
2919                  */
2920                 CFISCSI_SESSION_LOCK(cs);
2921                 TAILQ_FOREACH_SAFE(cdw,
2922                     &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2923                         if (bhstmr->bhstmr_referenced_task_tag !=
2924                             cdw->cdw_initiator_task_tag)
2925                                 continue;
2926
2927 #if 0
2928                         CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2929                             "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2930 #endif
2931                         TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2932                             cdw, cdw_next);
2933                         io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
2934                         cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
2935                         cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2936                         cfiscsi_data_wait_free(cs, cdw);
2937                 }
2938                 CFISCSI_SESSION_UNLOCK(cs);
2939         }
2940         if ((bhstmr->bhstmr_function & ~0x80) ==
2941             BHSTMR_FUNCTION_TARGET_COLD_RESET &&
2942             io->io_hdr.status == CTL_SUCCESS)
2943                 cold_reset = 1;
2944
2945         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2946         bhstmr2 = (struct iscsi_bhs_task_management_response *)
2947             response->ip_bhs;
2948         bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2949         bhstmr2->bhstmr_flags = 0x80;
2950         switch (io->taskio.task_status) {
2951         case CTL_TASK_FUNCTION_COMPLETE:
2952                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2953                 break;
2954         case CTL_TASK_FUNCTION_SUCCEEDED:
2955                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
2956                 break;
2957         case CTL_TASK_LUN_DOES_NOT_EXIST:
2958                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
2959                 break;
2960         case CTL_TASK_FUNCTION_NOT_SUPPORTED:
2961         default:
2962                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2963                 break;
2964         }
2965         memcpy(bhstmr2->bhstmr_additional_reponse_information,
2966             io->taskio.task_resp, sizeof(io->taskio.task_resp));
2967         bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2968
2969         ctl_free_io(io);
2970         icl_pdu_free(request);
2971         cfiscsi_pdu_queue(response);
2972
2973         if (cold_reset) {
2974                 softc = cs->cs_target->ct_softc;
2975                 mtx_lock(&softc->lock);
2976                 TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
2977                         if (tcs->cs_target == cs->cs_target)
2978                                 cfiscsi_session_terminate(tcs);
2979                 }
2980                 mtx_unlock(&softc->lock);
2981         }
2982 }
2983
2984 static void
2985 cfiscsi_done(union ctl_io *io)
2986 {
2987         struct icl_pdu *request;
2988         struct cfiscsi_session *cs;
2989
2990         KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2991                 ("invalid CTL status %#x", io->io_hdr.status));
2992
2993         if (io->io_hdr.io_type == CTL_IO_TASK &&
2994             io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
2995                 /*
2996                  * Implicit task termination has just completed; nothing to do.
2997                  */
2998                 cs = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2999                 cs->cs_tasks_aborted = true;
3000                 refcount_release(&cs->cs_outstanding_ctl_pdus);
3001                 wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
3002                 ctl_free_io(io);
3003                 return;
3004         }
3005
3006         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
3007         cs = PDU_SESSION(request);
3008
3009         switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3010         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3011                 cfiscsi_scsi_command_done(io);
3012                 break;
3013         case ISCSI_BHS_OPCODE_TASK_REQUEST:
3014                 cfiscsi_task_management_done(io);
3015                 break;
3016         default:
3017                 panic("cfiscsi_done called with wrong opcode 0x%x",
3018                     request->ip_bhs->bhs_opcode);
3019         }
3020
3021         refcount_release(&cs->cs_outstanding_ctl_pdus);
3022 }