sys/cam/ctl/ctl_frontend_iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 /*
  33  * CTL frontend for the iSCSI protocol.
  34  */
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include <sys/param.h>
  40 #include <sys/capsicum.h>
  41 #include <sys/condvar.h>
  42 #include <sys/file.h>
  43 #include <sys/kernel.h>
  44 #include <sys/kthread.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mutex.h>
  49 #include <sys/queue.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/systm.h>
  53 #include <sys/uio.h>
  54 #include <sys/unistd.h>
  55 #include <vm/uma.h>
  56
  57 #include <cam/scsi/scsi_all.h>
  58 #include <cam/scsi/scsi_da.h>
  59 #include <cam/ctl/ctl_io.h>
  60 #include <cam/ctl/ctl.h>
  61 #include <cam/ctl/ctl_backend.h>
  62 #include <cam/ctl/ctl_error.h>
  63 #include <cam/ctl/ctl_frontend.h>
  64 #include <cam/ctl/ctl_frontend_internal.h>
  65 #include <cam/ctl/ctl_debug.h>
  66 #include <cam/ctl/ctl_ha.h>
  67 #include <cam/ctl/ctl_ioctl.h>
  68 #include <cam/ctl/ctl_private.h>
  69
  70 #include <dev/iscsi/icl.h>
  71 #include <dev/iscsi/iscsi_proto.h>
  72 #include <cam/ctl/ctl_frontend_iscsi.h>
  73
  74 #ifdef ICL_KERNEL_PROXY
  75 #include <sys/socketvar.h>
  76 #endif
  77
  78 #ifdef ICL_KERNEL_PROXY
  79 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
  80 #endif
  81
  82 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
  83 static uma_zone_t cfiscsi_data_wait_zone;
  84
  85 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
  86     "CAM Target Layer iSCSI Frontend");
  87 static int debug = 1;
  88 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  89     &debug, 1, "Enable debug messages");
  90 static int ping_timeout = 5;
  91 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
  92     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
  93 static int login_timeout = 60;
  94 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
  95     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
  96 static int maxcmdsn_delta = 256;
  97 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RWTUN,
  98     &maxcmdsn_delta, 256, "Number of commands the initiator can send "
  99     "without confirmation");
 100
 101 #define CFISCSI_DEBUG(X, ...)                                           \
 102         do {                                                            \
 103                 if (debug > 1) {                                        \
 104                         printf("%s: " X "\n",                           \
 105                             __func__, ## __VA_ARGS__);                  \
 106                 }                                                       \
 107         } while (0)
 108
 109 #define CFISCSI_WARN(X, ...)                                            \
 110         do {                                                            \
 111                 if (debug > 0) {                                        \
 112                         printf("WARNING: %s: " X "\n",                  \
 113                             __func__, ## __VA_ARGS__);                  \
 114                 }                                                       \
 115         } while (0)
 116
 117 #define CFISCSI_SESSION_DEBUG(S, X, ...)                                \
 118         do {                                                            \
 119                 if (debug > 1) {                                        \
 120                         printf("%s: %s (%s): " X "\n",                  \
 121                             __func__, S->cs_initiator_addr,             \
 122                             S->cs_initiator_name, ## __VA_ARGS__);      \
 123                 }                                                       \
 124         } while (0)
 125
 126 #define CFISCSI_SESSION_WARN(S, X, ...)                                 \
 127         do  {                                                           \
 128                 if (debug > 0) {                                        \
 129                         printf("WARNING: %s (%s): " X "\n",             \
 130                             S->cs_initiator_addr,                       \
 131                             S->cs_initiator_name, ## __VA_ARGS__);      \
 132                 }                                                       \
 133         } while (0)
 134
 135 #define CFISCSI_SESSION_LOCK(X)         mtx_lock(&X->cs_lock)
 136 #define CFISCSI_SESSION_UNLOCK(X)       mtx_unlock(&X->cs_lock)
 137 #define CFISCSI_SESSION_LOCK_ASSERT(X)  mtx_assert(&X->cs_lock, MA_OWNED)
 138
 139 #define CONN_SESSION(X)                 ((struct cfiscsi_session *)(X)->ic_prv0)
 140 #define PDU_SESSION(X)                  CONN_SESSION((X)->ip_conn)
 141 #define PDU_EXPDATASN(X)                (X)->ip_prv0
 142 #define PDU_TOTAL_TRANSFER_LEN(X)       (X)->ip_prv1
 143 #define PDU_R2TSN(X)                    (X)->ip_prv2
 144
 145 int             cfiscsi_init(void);
 146 static void     cfiscsi_online(void *arg);
 147 static void     cfiscsi_offline(void *arg);
 148 static int      cfiscsi_info(void *arg, struct sbuf *sb);
 149 static int      cfiscsi_lun_enable(void *arg,
 150                     struct ctl_id target_id, int lun_id);
 151 static int      cfiscsi_lun_disable(void *arg,
 152                     struct ctl_id target_id, int lun_id);
 153 static uint32_t cfiscsi_lun_map(void *arg, uint32_t lun);
 154 static int      cfiscsi_ioctl(struct cdev *dev,
 155                     u_long cmd, caddr_t addr, int flag, struct thread *td);
 156 static void     cfiscsi_datamove(union ctl_io *io);
 157 static void     cfiscsi_datamove_in(union ctl_io *io);
 158 static void     cfiscsi_datamove_out(union ctl_io *io);
 159 static void     cfiscsi_done(union ctl_io *io);
 160 static bool     cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
 161 static void     cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
 162 static void     cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
 163 static void     cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
 164 static void     cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
 165 static void     cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
 166 static void     cfiscsi_session_terminate(struct cfiscsi_session *cs);
 167 static struct cfiscsi_target    *cfiscsi_target_find(struct cfiscsi_softc
 168                     *softc, const char *name);
 169 static struct cfiscsi_target    *cfiscsi_target_find_or_create(
 170     struct cfiscsi_softc *softc, const char *name, const char *alias);
 171 static void     cfiscsi_target_release(struct cfiscsi_target *ct);
 172 static void     cfiscsi_session_delete(struct cfiscsi_session *cs);
 173
 174 static struct cfiscsi_softc cfiscsi_softc;
 175 extern struct ctl_softc *control_softc;
 176
 177 static struct ctl_frontend cfiscsi_frontend =
 178 {
 179         .name = "iscsi",
 180         .init = cfiscsi_init,
 181         .ioctl = cfiscsi_ioctl,
 182 };
 183 CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend);
 184 MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1);
 185
 186 static struct icl_pdu *
 187 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
 188 {
 189
 190         return (icl_pdu_new(request->ip_conn, flags));
 191 }
 192
 193 static bool
 194 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
 195 {
 196         const struct iscsi_bhs_scsi_command *bhssc;
 197         struct cfiscsi_session *cs;
 198         uint32_t cmdsn, expstatsn;
 199
 200         cs = PDU_SESSION(request);
 201
 202         /*
 203          * Every incoming PDU - not just NOP-Out - resets the ping timer.
 204          * The purpose of the timeout is to reset the connection when it stalls;
 205          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 206          * in some queue.
 207          *
 208          * XXX: Locking?
 209          */
 210         cs->cs_timeout = 0;
 211
 212         /*
 213          * Data-Out PDUs don't contain CmdSN.
 214          */
 215         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 216             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 217                 return (false);
 218
 219         /*
 220          * We're only using fields common for all the request
 221          * (initiator -> target) PDUs.
 222          */
 223         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
 224         cmdsn = ntohl(bhssc->bhssc_cmdsn);
 225         expstatsn = ntohl(bhssc->bhssc_expstatsn);
 226
 227         CFISCSI_SESSION_LOCK(cs);
 228 #if 0
 229         if (expstatsn != cs->cs_statsn) {
 230                 CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
 231                     "while current StatSN is %d", expstatsn,
 232                     cs->cs_statsn);
 233         }
 234 #endif
 235
 236         /*
 237          * The target MUST silently ignore any non-immediate command outside
 238          * of this range.
 239          */
 240         if (cmdsn < cs->cs_cmdsn || cmdsn > cs->cs_cmdsn + maxcmdsn_delta) {
 241                 CFISCSI_SESSION_UNLOCK(cs);
 242                 CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %d, "
 243                     "while expected CmdSN was %d", cmdsn, cs->cs_cmdsn);
 244                 return (true);
 245         }
 246
 247         if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 248                 cs->cs_cmdsn++;
 249
 250         CFISCSI_SESSION_UNLOCK(cs);
 251
 252         return (false);
 253 }
 254
 255 static void
 256 cfiscsi_pdu_handle(struct icl_pdu *request)
 257 {
 258         struct cfiscsi_session *cs;
 259         bool ignore;
 260
 261         cs = PDU_SESSION(request);
 262
 263         ignore = cfiscsi_pdu_update_cmdsn(request);
 264         if (ignore) {
 265                 icl_pdu_free(request);
 266                 return;
 267         }
 268
 269         /*
 270          * Handle the PDU; this includes e.g. receiving the remaining
 271          * part of PDU and submitting the SCSI command to CTL
 272          * or queueing a reply.  The handling routine is responsible
 273          * for freeing the PDU when it's no longer needed.
 274          */
 275         switch (request->ip_bhs->bhs_opcode &
 276             ~ISCSI_BHS_OPCODE_IMMEDIATE) {
 277         case ISCSI_BHS_OPCODE_NOP_OUT:
 278                 cfiscsi_pdu_handle_nop_out(request);
 279                 break;
 280         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
 281                 cfiscsi_pdu_handle_scsi_command(request);
 282                 break;
 283         case ISCSI_BHS_OPCODE_TASK_REQUEST:
 284                 cfiscsi_pdu_handle_task_request(request);
 285                 break;
 286         case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
 287                 cfiscsi_pdu_handle_data_out(request);
 288                 break;
 289         case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
 290                 cfiscsi_pdu_handle_logout_request(request);
 291                 break;
 292         default:
 293                 CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
 294                     "opcode 0x%x; dropping connection",
 295                     request->ip_bhs->bhs_opcode);
 296                 icl_pdu_free(request);
 297                 cfiscsi_session_terminate(cs);
 298         }
 299
 300 }
 301
 302 static void
 303 cfiscsi_receive_callback(struct icl_pdu *request)
 304 {
 305         struct cfiscsi_session *cs;
 306
 307         cs = PDU_SESSION(request);
 308
 309 #ifdef ICL_KERNEL_PROXY
 310         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 311                 if (cs->cs_login_pdu == NULL)
 312                         cs->cs_login_pdu = request;
 313                 else
 314                         icl_pdu_free(request);
 315                 cv_signal(&cs->cs_login_cv);
 316                 return;
 317         }
 318 #endif
 319
 320         cfiscsi_pdu_handle(request);
 321 }
 322
 323 static void
 324 cfiscsi_error_callback(struct icl_conn *ic)
 325 {
 326         struct cfiscsi_session *cs;
 327
 328         cs = CONN_SESSION(ic);
 329
 330         CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
 331         cfiscsi_session_terminate(cs);
 332 }
 333
 334 static int
 335 cfiscsi_pdu_prepare(struct icl_pdu *response)
 336 {
 337         struct cfiscsi_session *cs;
 338         struct iscsi_bhs_scsi_response *bhssr;
 339         bool advance_statsn = true;
 340
 341         cs = PDU_SESSION(response);
 342
 343         CFISCSI_SESSION_LOCK_ASSERT(cs);
 344
 345         /*
 346          * We're only using fields common for all the response
 347          * (target -> initiator) PDUs.
 348          */
 349         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 350
 351         /*
 352          * 10.8.3: "The StatSN for this connection is not advanced
 353          * after this PDU is sent."
 354          */
 355         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
 356                 advance_statsn = false;
 357
 358         /*
 359          * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
 360          * StatSN for the connection is not advanced after this PDU is sent."
 361          */
 362         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
 363             bhssr->bhssr_initiator_task_tag == 0xffffffff)
 364                 advance_statsn = false;
 365
 366         /*
 367          * See the comment below - StatSN is not meaningful and must
 368          * not be advanced.
 369          */
 370         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
 371             (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
 372                 advance_statsn = false;
 373
 374         /*
 375          * 10.7.3: "The fields StatSN, Status, and Residual Count
 376          * only have meaningful content if the S bit is set to 1."
 377          */
 378         if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
 379             (bhssr->bhssr_flags & BHSDI_FLAGS_S))
 380                 bhssr->bhssr_statsn = htonl(cs->cs_statsn);
 381         bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
 382         bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn + maxcmdsn_delta);
 383
 384         if (advance_statsn)
 385                 cs->cs_statsn++;
 386
 387         return (0);
 388 }
 389
 390 static void
 391 cfiscsi_pdu_queue(struct icl_pdu *response)
 392 {
 393         struct cfiscsi_session *cs;
 394
 395         cs = PDU_SESSION(response);
 396
 397         CFISCSI_SESSION_LOCK(cs);
 398         cfiscsi_pdu_prepare(response);
 399         icl_pdu_queue(response);
 400         CFISCSI_SESSION_UNLOCK(cs);
 401 }
 402
 403 static uint32_t
 404 cfiscsi_decode_lun(uint64_t encoded)
 405 {
 406         uint8_t lun[8];
 407         uint32_t result;
 408
 409         /*
 410          * The LUN field in iSCSI PDUs may look like an ordinary 64 bit number,
 411          * but is in fact an evil, multidimensional structure defined
 412          * in SCSI Architecture Model 5 (SAM-5), section 4.6.
 413          */
 414         memcpy(lun, &encoded, sizeof(lun));
 415         switch (lun[0] & 0xC0) {
 416         case 0x00:
 417                 if ((lun[0] & 0x3f) != 0 || lun[2] != 0 || lun[3] != 0 ||
 418                     lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) {
 419                         CFISCSI_WARN("malformed LUN "
 420                             "(peripheral device addressing method): 0x%jx",
 421                             (uintmax_t)encoded);
 422                         result = 0xffffffff;
 423                         break;
 424                 }
 425                 result = lun[1];
 426                 break;
 427         case 0x40:
 428                 if (lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 ||
 429                     lun[6] != 0 || lun[7] != 0) {
 430                         CFISCSI_WARN("malformed LUN "
 431                             "(flat address space addressing method): 0x%jx",
 432                             (uintmax_t)encoded);
 433                         result = 0xffffffff;
 434                         break;
 435                 }
 436                 result = ((lun[0] & 0x3f) << 8) + lun[1];
 437                 break;
 438         case 0xC0:
 439                 if (lun[0] != 0xD2 || lun[4] != 0 || lun[5] != 0 ||
 440                     lun[6] != 0 || lun[7] != 0) {
 441                         CFISCSI_WARN("malformed LUN (extended flat "
 442                             "address space addressing method): 0x%jx",
 443                             (uintmax_t)encoded);
 444                         result = 0xffffffff;
 445                         break;
 446                 }
 447                 result = (lun[1] << 16) + (lun[2] << 8) + lun[3];
 448         default:
 449                 CFISCSI_WARN("unsupported LUN format 0x%jx",
 450                     (uintmax_t)encoded);
 451                 result = 0xffffffff;
 452                 break;
 453         }
 454
 455         return (result);
 456 }
 457
 458 static void
 459 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
 460 {
 461         struct cfiscsi_session *cs;
 462         struct iscsi_bhs_nop_out *bhsno;
 463         struct iscsi_bhs_nop_in *bhsni;
 464         struct icl_pdu *response;
 465         void *data = NULL;
 466         size_t datasize;
 467         int error;
 468
 469         cs = PDU_SESSION(request);
 470         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 471
 472         if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
 473                 /*
 474                  * Nothing to do, iscsi_pdu_update_statsn() already
 475                  * zeroed the timeout.
 476                  */
 477                 icl_pdu_free(request);
 478                 return;
 479         }
 480
 481         datasize = icl_pdu_data_segment_length(request);
 482         if (datasize > 0) {
 483                 data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
 484                 if (data == NULL) {
 485                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 486                             "dropping connection");
 487                         icl_pdu_free(request);
 488                         cfiscsi_session_terminate(cs);
 489                         return;
 490                 }
 491                 icl_pdu_get_data(request, 0, data, datasize);
 492         }
 493
 494         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 495         if (response == NULL) {
 496                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 497                     "droppping connection");
 498                 free(data, M_CFISCSI);
 499                 icl_pdu_free(request);
 500                 cfiscsi_session_terminate(cs);
 501                 return;
 502         }
 503         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 504         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
 505         bhsni->bhsni_flags = 0x80;
 506         bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
 507         bhsni->bhsni_target_transfer_tag = 0xffffffff;
 508         if (datasize > 0) {
 509                 error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
 510                 if (error != 0) {
 511                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 512                             "dropping connection");
 513                         free(data, M_CFISCSI);
 514                         icl_pdu_free(request);
 515                         icl_pdu_free(response);
 516                         cfiscsi_session_terminate(cs);
 517                         return;
 518                 }
 519                 free(data, M_CFISCSI);
 520         }
 521
 522         icl_pdu_free(request);
 523         cfiscsi_pdu_queue(response);
 524 }
 525
 526 static void
 527 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
 528 {
 529         struct iscsi_bhs_scsi_command *bhssc;
 530         struct cfiscsi_session *cs;
 531         union ctl_io *io;
 532         int error;
 533
 534         cs = PDU_SESSION(request);
 535         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 536         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
 537         //    bhssc->bhssc_initiator_task_tag);
 538
 539         if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
 540                 CFISCSI_SESSION_WARN(cs, "unsolicited data with "
 541                     "ImmediateData=No; dropping connection");
 542                 icl_pdu_free(request);
 543                 cfiscsi_session_terminate(cs);
 544                 return;
 545         }
 546         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
 547         ctl_zero_io(io);
 548         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 549         io->io_hdr.io_type = CTL_IO_SCSI;
 550         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 551         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
 552         io->io_hdr.nexus.targ_target.id = 0;
 553         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhssc->bhssc_lun);
 554         io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
 555         switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
 556         case BHSSC_FLAGS_ATTR_UNTAGGED:
 557                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 558                 break;
 559         case BHSSC_FLAGS_ATTR_SIMPLE:
 560                 io->scsiio.tag_type = CTL_TAG_SIMPLE;
 561                 break;
 562         case BHSSC_FLAGS_ATTR_ORDERED:
 563                 io->scsiio.tag_type = CTL_TAG_ORDERED;
 564                 break;
 565         case BHSSC_FLAGS_ATTR_HOQ:
 566                 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
 567                 break;
 568         case BHSSC_FLAGS_ATTR_ACA:
 569                 io->scsiio.tag_type = CTL_TAG_ACA;
 570                 break;
 571         default:
 572                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 573                 CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
 574                     bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
 575                 break;
 576         }
 577         io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
 578         memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
 579         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 580         error = ctl_queue(io);
 581         if (error != CTL_RETVAL_COMPLETE) {
 582                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 583                     "dropping connection", error);
 584                 ctl_free_io(io);
 585                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 586                 icl_pdu_free(request);
 587                 cfiscsi_session_terminate(cs);
 588         }
 589 }
 590
 591 static void
 592 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
 593 {
 594         struct iscsi_bhs_task_management_request *bhstmr;
 595         struct iscsi_bhs_task_management_response *bhstmr2;
 596         struct icl_pdu *response;
 597         struct cfiscsi_session *cs;
 598         union ctl_io *io;
 599         int error;
 600
 601         cs = PDU_SESSION(request);
 602         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
 603         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
 604         ctl_zero_io(io);
 605         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 606         io->io_hdr.io_type = CTL_IO_TASK;
 607         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 608         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
 609         io->io_hdr.nexus.targ_target.id = 0;
 610         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhstmr->bhstmr_lun);
 611         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
 612
 613         switch (bhstmr->bhstmr_function & ~0x80) {
 614         case BHSTMR_FUNCTION_ABORT_TASK:
 615 #if 0
 616                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
 617 #endif
 618                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
 619                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
 620                 break;
 621         case BHSTMR_FUNCTION_ABORT_TASK_SET:
 622 #if 0
 623                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
 624 #endif
 625                 io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
 626                 break;
 627         case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
 628 #if 0
 629                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
 630 #endif
 631                 io->taskio.task_action = CTL_TASK_LUN_RESET;
 632                 break;
 633         case BHSTMR_FUNCTION_TARGET_WARM_RESET:
 634 #if 0
 635                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
 636 #endif
 637                 io->taskio.task_action = CTL_TASK_TARGET_RESET;
 638                 break;
 639         default:
 640                 CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
 641                     bhstmr->bhstmr_function & ~0x80);
 642                 ctl_free_io(io);
 643
 644                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 645                 if (response == NULL) {
 646                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 647                             "dropping connection");
 648                         icl_pdu_free(request);
 649                         cfiscsi_session_terminate(cs);
 650                         return;
 651                 }
 652                 bhstmr2 = (struct iscsi_bhs_task_management_response *)
 653                     response->ip_bhs;
 654                 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
 655                 bhstmr2->bhstmr_flags = 0x80;
 656                 bhstmr2->bhstmr_response =
 657                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
 658                 bhstmr2->bhstmr_initiator_task_tag =
 659                     bhstmr->bhstmr_initiator_task_tag;
 660                 icl_pdu_free(request);
 661                 cfiscsi_pdu_queue(response);
 662                 return;
 663         }
 664
 665         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 666         error = ctl_queue(io);
 667         if (error != CTL_RETVAL_COMPLETE) {
 668                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 669                     "dropping connection", error);
 670                 ctl_free_io(io);
 671                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 672                 icl_pdu_free(request);
 673                 cfiscsi_session_terminate(cs);
 674         }
 675 }
 676
 677 static bool
 678 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
 679 {
 680         struct iscsi_bhs_data_out *bhsdo;
 681         struct cfiscsi_session *cs;
 682         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
 683         size_t copy_len, len, off, buffer_offset;
 684         int ctl_sg_count;
 685         union ctl_io *io;
 686
 687         cs = PDU_SESSION(request);
 688
 689         KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 690             ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
 691             (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 692             ISCSI_BHS_OPCODE_SCSI_COMMAND,
 693             ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
 694
 695         /*
 696          * We're only using fields common for Data-Out and SCSI Command PDUs.
 697          */
 698         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 699
 700         io = cdw->cdw_ctl_io;
 701         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 702             ("CTL_FLAG_DATA_IN"));
 703
 704 #if 0
 705         CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
 706             request->ip_data_len, io->scsiio.kern_total_len);
 707 #endif
 708
 709         if (io->scsiio.kern_sg_entries > 0) {
 710                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
 711                 ctl_sg_count = io->scsiio.kern_sg_entries;
 712         } else {
 713                 ctl_sglist = &ctl_sg_entry;
 714                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
 715                 ctl_sglist->len = io->scsiio.kern_data_len;
 716                 ctl_sg_count = 1;
 717         }
 718
 719         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 720             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 721                 buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
 722         else
 723                 buffer_offset = 0;
 724         len = icl_pdu_data_segment_length(request);
 725
 726         /*
 727          * Make sure the offset, as sent by the initiator, matches the offset
 728          * we're supposed to be at in the scatter-gather list.
 729          */
 730         if (buffer_offset >
 731             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
 732             buffer_offset + len <=
 733             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
 734                 CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
 735                     "expected %zd; dropping connection", buffer_offset,
 736                     (size_t)io->scsiio.kern_rel_offset +
 737                     (size_t)io->scsiio.ext_data_filled);
 738                 ctl_set_data_phase_error(&io->scsiio);
 739                 cfiscsi_session_terminate(cs);
 740                 return (true);
 741         }
 742
 743         /*
 744          * This is the offset within the PDU data segment, as opposed
 745          * to buffer_offset, which is the offset within the task (SCSI
 746          * command).
 747          */
 748         off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
 749             buffer_offset;
 750
 751         /*
 752          * Iterate over the scatter/gather segments, filling them with data
 753          * from the PDU data segment.  Note that this can get called multiple
 754          * times for one SCSI command; the cdw structure holds state for the
 755          * scatter/gather list.
 756          */
 757         for (;;) {
 758                 KASSERT(cdw->cdw_sg_index < ctl_sg_count,
 759                     ("cdw->cdw_sg_index >= ctl_sg_count"));
 760                 if (cdw->cdw_sg_len == 0) {
 761                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
 762                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
 763                 }
 764                 KASSERT(off <= len, ("len > off"));
 765                 copy_len = len - off;
 766                 if (copy_len > cdw->cdw_sg_len)
 767                         copy_len = cdw->cdw_sg_len;
 768
 769                 icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
 770                 cdw->cdw_sg_addr += copy_len;
 771                 cdw->cdw_sg_len -= copy_len;
 772                 off += copy_len;
 773                 io->scsiio.ext_data_filled += copy_len;
 774
 775                 if (cdw->cdw_sg_len == 0) {
 776                         /*
 777                          * End of current segment.
 778                          */
 779                         if (cdw->cdw_sg_index == ctl_sg_count - 1) {
 780                                 /*
 781                                  * Last segment in scatter/gather list.
 782                                  */
 783                                 break;
 784                         }
 785                         cdw->cdw_sg_index++;
 786                 }
 787
 788                 if (off == len) {
 789                         /*
 790                          * End of PDU payload.
 791                          */
 792                         break;
 793                 }
 794         }
 795
 796         if (len > off) {
 797                 /*
 798                  * In case of unsolicited data, it's possible that the buffer
 799                  * provided by CTL is smaller than negotiated FirstBurstLength.
 800                  * Just ignore the superfluous data; will ask for them with R2T
 801                  * on next call to cfiscsi_datamove().
 802                  *
 803                  * This obviously can only happen with SCSI Command PDU.
 804                  */
 805                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 806                     ISCSI_BHS_OPCODE_SCSI_COMMAND)
 807                         return (true);
 808
 809                 CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
 810                     "expected %zd; dropping connection",
 811                     icl_pdu_data_segment_length(request), off);
 812                 ctl_set_data_phase_error(&io->scsiio);
 813                 cfiscsi_session_terminate(cs);
 814                 return (true);
 815         }
 816
 817         if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
 818             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
 819                 CFISCSI_SESSION_WARN(cs, "got the final packet without "
 820                     "the F flag; flags = 0x%x; dropping connection",
 821                     bhsdo->bhsdo_flags);
 822                 ctl_set_data_phase_error(&io->scsiio);
 823                 cfiscsi_session_terminate(cs);
 824                 return (true);
 825         }
 826
 827         if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
 828             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
 829                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 830                     ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 831                         CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
 832                             "transmitted size was %zd bytes instead of %d; "
 833                             "dropping connection",
 834                             (size_t)io->scsiio.ext_data_filled,
 835                             cdw->cdw_r2t_end);
 836                         ctl_set_data_phase_error(&io->scsiio);
 837                         cfiscsi_session_terminate(cs);
 838                         return (true);
 839                 } else {
 840                         /*
 841                          * For SCSI Command PDU, this just means we need to
 842                          * solicit more data by sending R2T.
 843                          */
 844                         return (false);
 845                 }
 846         }
 847
 848         if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
 849 #if 0
 850                 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
 851                     "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
 852 #endif
 853
 854                 return (true);
 855         }
 856
 857         return (false);
 858 }
 859
 860 static void
 861 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
 862 {
 863         struct iscsi_bhs_data_out *bhsdo;
 864         struct cfiscsi_session *cs;
 865         struct cfiscsi_data_wait *cdw = NULL;
 866         union ctl_io *io;
 867         bool done;
 868
 869         cs = PDU_SESSION(request);
 870         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 871
 872         CFISCSI_SESSION_LOCK(cs);
 873         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
 874 #if 0
 875                 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
 876                     "ttt 0x%x, itt 0x%x",
 877                     bhsdo->bhsdo_target_transfer_tag,
 878                     bhsdo->bhsdo_initiator_task_tag,
 879                     cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
 880 #endif
 881                 if (bhsdo->bhsdo_target_transfer_tag ==
 882                     cdw->cdw_target_transfer_tag)
 883                         break;
 884         }
 885         CFISCSI_SESSION_UNLOCK(cs);
 886         if (cdw == NULL) {
 887                 CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
 888                     "0x%x, not found; dropping connection",
 889                     bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
 890                 icl_pdu_free(request);
 891                 cfiscsi_session_terminate(cs);
 892                 return;
 893         }
 894
 895         io = cdw->cdw_ctl_io;
 896         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 897             ("CTL_FLAG_DATA_IN"));
 898
 899         done = cfiscsi_handle_data_segment(request, cdw);
 900         if (done) {
 901                 CFISCSI_SESSION_LOCK(cs);
 902                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
 903                 CFISCSI_SESSION_UNLOCK(cs);
 904                 done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
 905                     io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
 906                 uma_zfree(cfiscsi_data_wait_zone, cdw);
 907                 if (done)
 908                         io->scsiio.be_move_done(io);
 909                 else
 910                         cfiscsi_datamove_out(io);
 911         }
 912
 913         icl_pdu_free(request);
 914 }
 915
 916 static void
 917 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
 918 {
 919         struct iscsi_bhs_logout_request *bhslr;
 920         struct iscsi_bhs_logout_response *bhslr2;
 921         struct icl_pdu *response;
 922         struct cfiscsi_session *cs;
 923
 924         cs = PDU_SESSION(request);
 925         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 926         switch (bhslr->bhslr_reason & 0x7f) {
 927         case BHSLR_REASON_CLOSE_SESSION:
 928         case BHSLR_REASON_CLOSE_CONNECTION:
 929                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 930                 if (response == NULL) {
 931                         CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
 932                         icl_pdu_free(request);
 933                         cfiscsi_session_terminate(cs);
 934                         return;
 935                 }
 936                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 937                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 938                 bhslr2->bhslr_flags = 0x80;
 939                 bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
 940                 bhslr2->bhslr_initiator_task_tag =
 941                     bhslr->bhslr_initiator_task_tag;
 942                 icl_pdu_free(request);
 943                 cfiscsi_pdu_queue(response);
 944                 cfiscsi_session_terminate(cs);
 945                 break;
 946         case BHSLR_REASON_REMOVE_FOR_RECOVERY:
 947                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 948                 if (response == NULL) {
 949                         CFISCSI_SESSION_WARN(cs,
 950                             "failed to allocate memory; dropping connection");
 951                         icl_pdu_free(request);
 952                         cfiscsi_session_terminate(cs);
 953                         return;
 954                 }
 955                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 956                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 957                 bhslr2->bhslr_flags = 0x80;
 958                 bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
 959                 bhslr2->bhslr_initiator_task_tag =
 960                     bhslr->bhslr_initiator_task_tag;
 961                 icl_pdu_free(request);
 962                 cfiscsi_pdu_queue(response);
 963                 break;
 964         default:
 965                 CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
 966                     bhslr->bhslr_reason);
 967                 icl_pdu_free(request);
 968                 cfiscsi_session_terminate(cs);
 969                 break;
 970         }
 971 }
 972
 973 static void
 974 cfiscsi_callout(void *context)
 975 {
 976         struct icl_pdu *cp;
 977         struct iscsi_bhs_nop_in *bhsni;
 978         struct cfiscsi_session *cs;
 979
 980         cs = context;
 981
 982         if (cs->cs_terminating)
 983                 return;
 984
 985         callout_schedule(&cs->cs_callout, 1 * hz);
 986
 987         atomic_add_int(&cs->cs_timeout, 1);
 988
 989 #ifdef ICL_KERNEL_PROXY
 990         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 991                 if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
 992                         CFISCSI_SESSION_WARN(cs, "login timed out after "
 993                             "%d seconds; dropping connection", cs->cs_timeout);
 994                         cfiscsi_session_terminate(cs);
 995                 }
 996                 return;
 997         }
 998 #endif
 999
1000         if (ping_timeout <= 0) {
1001                 /*
1002                  * Pings are disabled.  Don't send NOP-In in this case;
1003                  * user might have disabled pings to work around problems
1004                  * with certain initiators that can't properly handle
1005                  * NOP-In, such as iPXE.  Reset the timeout, to avoid
1006                  * triggering reconnection, should the user decide to
1007                  * reenable them.
1008                  */
1009                 cs->cs_timeout = 0;
1010                 return;
1011         }
1012
1013         if (cs->cs_timeout >= ping_timeout) {
1014                 CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1015                     "dropping connection",  ping_timeout);
1016                 cfiscsi_session_terminate(cs);
1017                 return;
1018         }
1019
1020         /*
1021          * If the ping was reset less than one second ago - which means
1022          * that we've received some PDU during the last second - assume
1023          * the traffic flows correctly and don't bother sending a NOP-Out.
1024          *
1025          * (It's 2 - one for one second, and one for incrementing is_timeout
1026          * earlier in this routine.)
1027          */
1028         if (cs->cs_timeout < 2)
1029                 return;
1030
1031         cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1032         if (cp == NULL) {
1033                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1034                 return;
1035         }
1036         bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1037         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1038         bhsni->bhsni_flags = 0x80;
1039         bhsni->bhsni_initiator_task_tag = 0xffffffff;
1040
1041         cfiscsi_pdu_queue(cp);
1042 }
1043
1044 static void
1045 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1046 {
1047         struct cfiscsi_data_wait *cdw;
1048         union ctl_io *io;
1049         int error, last, wait;
1050
1051         if (cs->cs_target == NULL)
1052                 return;         /* No target yet, so nothing to do. */
1053         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1054         ctl_zero_io(io);
1055         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = cs;
1056         io->io_hdr.io_type = CTL_IO_TASK;
1057         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1058         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1059         io->io_hdr.nexus.targ_target.id = 0;
1060         io->io_hdr.nexus.targ_lun = 0;
1061         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1062         io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1063         wait = cs->cs_outstanding_ctl_pdus;
1064         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1065         error = ctl_queue(io);
1066         if (error != CTL_RETVAL_COMPLETE) {
1067                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1068                 refcount_release(&cs->cs_outstanding_ctl_pdus);
1069                 ctl_free_io(io);
1070         }
1071
1072         CFISCSI_SESSION_LOCK(cs);
1073         while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1074                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1075                 CFISCSI_SESSION_UNLOCK(cs);
1076                 /*
1077                  * Set nonzero port status; this prevents backends from
1078                  * assuming that the data transfer actually succeeded
1079                  * and writing uninitialized data to disk.
1080                  */
1081                 cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1082                 cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1083                 uma_zfree(cfiscsi_data_wait_zone, cdw);
1084                 CFISCSI_SESSION_LOCK(cs);
1085         }
1086         CFISCSI_SESSION_UNLOCK(cs);
1087
1088         /*
1089          * Wait for CTL to terminate all the tasks.
1090          */
1091         if (wait > 0)
1092                 CFISCSI_SESSION_WARN(cs,
1093                     "waiting for CTL to terminate %d tasks", wait);
1094         for (;;) {
1095                 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1096                 last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1097                 if (last != 0)
1098                         break;
1099                 tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1100                     0, "cfiscsi_terminate", hz / 100);
1101         }
1102         if (wait > 0)
1103                 CFISCSI_SESSION_WARN(cs, "tasks terminated");
1104 }
1105
1106 static void
1107 cfiscsi_maintenance_thread(void *arg)
1108 {
1109         struct cfiscsi_session *cs;
1110
1111         cs = arg;
1112
1113         for (;;) {
1114                 CFISCSI_SESSION_LOCK(cs);
1115                 if (cs->cs_terminating == false)
1116                         cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1117                 CFISCSI_SESSION_UNLOCK(cs);
1118
1119                 if (cs->cs_terminating) {
1120
1121                         /*
1122                          * We used to wait up to 30 seconds to deliver queued
1123                          * PDUs to the initiator.  We also tried hard to deliver
1124                          * SCSI Responses for the aborted PDUs.  We don't do
1125                          * that anymore.  We might need to revisit that.
1126                          */
1127                         callout_drain(&cs->cs_callout);
1128                         icl_conn_close(cs->cs_conn);
1129
1130                         /*
1131                          * At this point ICL receive thread is no longer
1132                          * running; no new tasks can be queued.
1133                          */
1134                         cfiscsi_session_terminate_tasks(cs);
1135                         cfiscsi_session_delete(cs);
1136                         kthread_exit();
1137                         return;
1138                 }
1139                 CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1140         }
1141 }
1142
1143 static void
1144 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1145 {
1146
1147         if (cs->cs_terminating)
1148                 return;
1149         cs->cs_terminating = true;
1150         cv_signal(&cs->cs_maintenance_cv);
1151 #ifdef ICL_KERNEL_PROXY
1152         cv_signal(&cs->cs_login_cv);
1153 #endif
1154 }
1155
1156 static int
1157 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1158 {
1159         struct cfiscsi_target *ct;
1160         char *name;
1161         int i;
1162
1163         KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1164
1165         ct = cs->cs_target;
1166         name = strdup(cs->cs_initiator_id, M_CTL);
1167         i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1168         if (i < 0) {
1169                 CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1170                     i);
1171                 cs->cs_ctl_initid = -1;
1172                 return (1);
1173         }
1174         cs->cs_ctl_initid = i;
1175 #if 0
1176         CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1177 #endif
1178
1179         return (0);
1180 }
1181
1182 static void
1183 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1184 {
1185         int error;
1186
1187         if (cs->cs_ctl_initid == -1)
1188                 return;
1189
1190         error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1191         if (error != 0) {
1192                 CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1193                     error);
1194         }
1195         cs->cs_ctl_initid = -1;
1196 }
1197
1198 static struct cfiscsi_session *
1199 cfiscsi_session_new(struct cfiscsi_softc *softc)
1200 {
1201         struct cfiscsi_session *cs;
1202         int error;
1203
1204         cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1205         if (cs == NULL) {
1206                 CFISCSI_WARN("malloc failed");
1207                 return (NULL);
1208         }
1209         cs->cs_ctl_initid = -1;
1210
1211         refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1212         TAILQ_INIT(&cs->cs_waiting_for_data_out);
1213         mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1214         cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1215 #ifdef ICL_KERNEL_PROXY
1216         cv_init(&cs->cs_login_cv, "cfiscsi_login");
1217 #endif
1218
1219         cs->cs_conn = icl_conn_new("cfiscsi", &cs->cs_lock);
1220         cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1221         cs->cs_conn->ic_error = cfiscsi_error_callback;
1222         cs->cs_conn->ic_prv0 = cs;
1223
1224         error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1225         if (error != 0) {
1226                 CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1227                 free(cs, M_CFISCSI);
1228                 return (NULL);
1229         }
1230
1231         mtx_lock(&softc->lock);
1232         cs->cs_id = ++softc->last_session_id;
1233         TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1234         mtx_unlock(&softc->lock);
1235
1236         /*
1237          * Start pinging the initiator.
1238          */
1239         callout_init(&cs->cs_callout, 1);
1240         callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1241
1242         return (cs);
1243 }
1244
1245 static void
1246 cfiscsi_session_delete(struct cfiscsi_session *cs)
1247 {
1248         struct cfiscsi_softc *softc;
1249
1250         softc = &cfiscsi_softc;
1251
1252         KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1253             ("destroying session with outstanding CTL pdus"));
1254         KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1255             ("destroying session with non-empty queue"));
1256
1257         cfiscsi_session_unregister_initiator(cs);
1258         if (cs->cs_target != NULL)
1259                 cfiscsi_target_release(cs->cs_target);
1260         icl_conn_close(cs->cs_conn);
1261         icl_conn_free(cs->cs_conn);
1262
1263         mtx_lock(&softc->lock);
1264         TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1265         cv_signal(&softc->sessions_cv);
1266         mtx_unlock(&softc->lock);
1267
1268         free(cs, M_CFISCSI);
1269 }
1270
1271 int
1272 cfiscsi_init(void)
1273 {
1274         struct cfiscsi_softc *softc;
1275         int retval;
1276
1277         softc = &cfiscsi_softc;
1278         retval = 0;
1279         bzero(softc, sizeof(*softc));
1280         mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1281
1282         cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1283 #ifdef ICL_KERNEL_PROXY
1284         cv_init(&softc->accept_cv, "cfiscsi_accept");
1285 #endif
1286         TAILQ_INIT(&softc->sessions);
1287         TAILQ_INIT(&softc->targets);
1288
1289         cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1290             sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1291             UMA_ALIGN_PTR, 0);
1292
1293         return (0);
1294 }
1295
1296 #ifdef ICL_KERNEL_PROXY
1297 static void
1298 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1299 {
1300         struct cfiscsi_session *cs;
1301
1302         cs = cfiscsi_session_new(&cfiscsi_softc);
1303         if (cs == NULL) {
1304                 CFISCSI_WARN("failed to create session");
1305                 return;
1306         }
1307
1308         icl_conn_handoff_sock(cs->cs_conn, so);
1309         cs->cs_initiator_sa = sa;
1310         cs->cs_portal_id = portal_id;
1311         cs->cs_waiting_for_ctld = true;
1312         cv_signal(&cfiscsi_softc.accept_cv);
1313 }
1314 #endif
1315
1316 static void
1317 cfiscsi_online(void *arg)
1318 {
1319         struct cfiscsi_softc *softc;
1320         struct cfiscsi_target *ct;
1321         int online;
1322
1323         ct = (struct cfiscsi_target *)arg;
1324         softc = ct->ct_softc;
1325
1326         mtx_lock(&softc->lock);
1327         if (ct->ct_online) {
1328                 mtx_unlock(&softc->lock);
1329                 return;
1330         }
1331         ct->ct_online = 1;
1332         online = softc->online++;
1333         mtx_unlock(&softc->lock);
1334         if (online > 0)
1335                 return;
1336
1337 #ifdef ICL_KERNEL_PROXY
1338         if (softc->listener != NULL)
1339                 icl_listen_free(softc->listener);
1340         softc->listener = icl_listen_new(cfiscsi_accept);
1341 #endif
1342 }
1343
1344 static void
1345 cfiscsi_offline(void *arg)
1346 {
1347         struct cfiscsi_softc *softc;
1348         struct cfiscsi_target *ct;
1349         struct cfiscsi_session *cs;
1350         int online;
1351
1352         ct = (struct cfiscsi_target *)arg;
1353         softc = ct->ct_softc;
1354
1355         mtx_lock(&softc->lock);
1356         if (!ct->ct_online) {
1357                 mtx_unlock(&softc->lock);
1358                 return;
1359         }
1360         ct->ct_online = 0;
1361         online = --softc->online;
1362
1363         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1364                 if (cs->cs_target == ct)
1365                         cfiscsi_session_terminate(cs);
1366         }
1367         do {
1368                 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1369                         if (cs->cs_target == ct)
1370                                 break;
1371                 }
1372                 if (cs != NULL)
1373                         cv_wait(&softc->sessions_cv, &softc->lock);
1374         } while (cs != NULL && ct->ct_online == 0);
1375         mtx_unlock(&softc->lock);
1376         if (online > 0)
1377                 return;
1378
1379 #ifdef ICL_KERNEL_PROXY
1380         icl_listen_free(softc->listener);
1381         softc->listener = NULL;
1382 #endif
1383 }
1384
1385 static int
1386 cfiscsi_info(void *arg, struct sbuf *sb)
1387 {
1388         struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1389         int retval;
1390
1391         retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1392             ct->ct_state);
1393         return (retval);
1394 }
1395
1396 static void
1397 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1398 {
1399         struct cfiscsi_softc *softc;
1400         struct cfiscsi_session *cs, *cs2;
1401         struct cfiscsi_target *ct;
1402         struct ctl_iscsi_handoff_params *cihp;
1403         int error;
1404
1405         cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1406         softc = &cfiscsi_softc;
1407
1408         CFISCSI_DEBUG("new connection from %s (%s) to %s",
1409             cihp->initiator_name, cihp->initiator_addr,
1410             cihp->target_name);
1411
1412         ct = cfiscsi_target_find(softc, cihp->target_name);
1413         if (ct == NULL) {
1414                 ci->status = CTL_ISCSI_ERROR;
1415                 snprintf(ci->error_str, sizeof(ci->error_str),
1416                     "%s: target not found", __func__);
1417                 return;
1418         }
1419
1420 #ifdef ICL_KERNEL_PROXY
1421         if (cihp->socket > 0 && cihp->connection_id > 0) {
1422                 snprintf(ci->error_str, sizeof(ci->error_str),
1423                     "both socket and connection_id set");
1424                 ci->status = CTL_ISCSI_ERROR;
1425                 cfiscsi_target_release(ct);
1426                 return;
1427         }
1428         if (cihp->socket == 0) {
1429                 mtx_lock(&cfiscsi_softc.lock);
1430                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1431                         if (cs->cs_id == cihp->connection_id)
1432                                 break;
1433                 }
1434                 if (cs == NULL) {
1435                         mtx_unlock(&cfiscsi_softc.lock);
1436                         snprintf(ci->error_str, sizeof(ci->error_str),
1437                             "connection not found");
1438                         ci->status = CTL_ISCSI_ERROR;
1439                         cfiscsi_target_release(ct);
1440                         return;
1441                 }
1442                 mtx_unlock(&cfiscsi_softc.lock);
1443         } else {
1444 #endif
1445                 cs = cfiscsi_session_new(softc);
1446                 if (cs == NULL) {
1447                         ci->status = CTL_ISCSI_ERROR;
1448                         snprintf(ci->error_str, sizeof(ci->error_str),
1449                             "%s: cfiscsi_session_new failed", __func__);
1450                         cfiscsi_target_release(ct);
1451                         return;
1452                 }
1453 #ifdef ICL_KERNEL_PROXY
1454         }
1455 #endif
1456
1457         /*
1458          * First PDU of Full Feature phase has the same CmdSN as the last
1459          * PDU from the Login Phase received from the initiator.  Thus,
1460          * the -1 below.
1461          */
1462         cs->cs_portal_group_tag = cihp->portal_group_tag;
1463         cs->cs_cmdsn = cihp->cmdsn;
1464         cs->cs_statsn = cihp->statsn;
1465         cs->cs_max_data_segment_length = cihp->max_recv_data_segment_length;
1466         cs->cs_max_burst_length = cihp->max_burst_length;
1467         cs->cs_immediate_data = !!cihp->immediate_data;
1468         if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1469                 cs->cs_conn->ic_header_crc32c = true;
1470         if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1471                 cs->cs_conn->ic_data_crc32c = true;
1472
1473         strlcpy(cs->cs_initiator_name,
1474             cihp->initiator_name, sizeof(cs->cs_initiator_name));
1475         strlcpy(cs->cs_initiator_addr,
1476             cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1477         strlcpy(cs->cs_initiator_alias,
1478             cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1479         memcpy(cs->cs_initiator_isid,
1480             cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1481         snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1482             "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1483             cihp->initiator_isid[0], cihp->initiator_isid[1],
1484             cihp->initiator_isid[2], cihp->initiator_isid[3],
1485             cihp->initiator_isid[4], cihp->initiator_isid[5]);
1486
1487         mtx_lock(&softc->lock);
1488         if (ct->ct_online == 0) {
1489                 mtx_unlock(&softc->lock);
1490                 cfiscsi_session_terminate(cs);
1491                 cfiscsi_target_release(ct);
1492                 ci->status = CTL_ISCSI_ERROR;
1493                 snprintf(ci->error_str, sizeof(ci->error_str),
1494                     "%s: port offline", __func__);
1495                 return;
1496         }
1497         cs->cs_target = ct;
1498         mtx_unlock(&softc->lock);
1499
1500         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1501 restart:
1502         if (!cs->cs_terminating) {
1503                 mtx_lock(&softc->lock);
1504                 TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1505                         if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1506                             cs->cs_target == cs2->cs_target &&
1507                             cs->cs_portal_group_tag == cs2->cs_portal_group_tag &&
1508                             strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1509                                 cfiscsi_session_terminate(cs2);
1510                                 mtx_unlock(&softc->lock);
1511                                 pause("cfiscsi_reinstate", 1);
1512                                 goto restart;
1513                         }
1514                 }
1515                 mtx_unlock(&softc->lock);
1516         }
1517
1518         /*
1519          * Register initiator with CTL.
1520          */
1521         cfiscsi_session_register_initiator(cs);
1522
1523 #ifdef ICL_KERNEL_PROXY
1524         if (cihp->socket > 0) {
1525 #endif
1526                 error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1527                 if (error != 0) {
1528                         cfiscsi_session_terminate(cs);
1529                         refcount_release(&cs->cs_outstanding_ctl_pdus);
1530                         ci->status = CTL_ISCSI_ERROR;
1531                         snprintf(ci->error_str, sizeof(ci->error_str),
1532                             "%s: icl_conn_handoff failed with error %d",
1533                             __func__, error);
1534                         return;
1535                 }
1536 #ifdef ICL_KERNEL_PROXY
1537         }
1538 #endif
1539
1540 #ifdef ICL_KERNEL_PROXY
1541         cs->cs_login_phase = false;
1542
1543         /*
1544          * First PDU of the Full Feature phase has likely already arrived.
1545          * We have to pick it up and execute properly.
1546          */
1547         if (cs->cs_login_pdu != NULL) {
1548                 CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1549                 cfiscsi_pdu_handle(cs->cs_login_pdu);
1550                 cs->cs_login_pdu = NULL;
1551         }
1552 #endif
1553
1554         refcount_release(&cs->cs_outstanding_ctl_pdus);
1555         ci->status = CTL_ISCSI_OK;
1556 }
1557
1558 static void
1559 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1560 {
1561         struct ctl_iscsi_list_params *cilp;
1562         struct cfiscsi_session *cs;
1563         struct cfiscsi_softc *softc;
1564         struct sbuf *sb;
1565         int error;
1566
1567         cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1568         softc = &cfiscsi_softc;
1569
1570         sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1571         if (sb == NULL) {
1572                 ci->status = CTL_ISCSI_ERROR;
1573                 snprintf(ci->error_str, sizeof(ci->error_str),
1574                     "Unable to allocate %d bytes for iSCSI session list",
1575                     cilp->alloc_len);
1576                 return;
1577         }
1578
1579         sbuf_printf(sb, "<ctlislist>\n");
1580         mtx_lock(&softc->lock);
1581         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1582 #ifdef ICL_KERNEL_PROXY
1583                 if (cs->cs_target == NULL)
1584                         continue;
1585 #endif
1586                 error = sbuf_printf(sb, "<connection id=\"%d\">"
1587                     "<initiator>%s</initiator>"
1588                     "<initiator_addr>%s</initiator_addr>"
1589                     "<initiator_alias>%s</initiator_alias>"
1590                     "<target>%s</target>"
1591                     "<target_alias>%s</target_alias>"
1592                     "<header_digest>%s</header_digest>"
1593                     "<data_digest>%s</data_digest>"
1594                     "<max_data_segment_length>%zd</max_data_segment_length>"
1595                     "<immediate_data>%d</immediate_data>"
1596                     "<iser>%d</iser>"
1597                     "</connection>\n",
1598                     cs->cs_id,
1599                     cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1600                     cs->cs_target->ct_name, cs->cs_target->ct_alias,
1601                     cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1602                     cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1603                     cs->cs_max_data_segment_length,
1604                     cs->cs_immediate_data,
1605                     cs->cs_conn->ic_iser);
1606                 if (error != 0)
1607                         break;
1608         }
1609         mtx_unlock(&softc->lock);
1610         error = sbuf_printf(sb, "</ctlislist>\n");
1611         if (error != 0) {
1612                 sbuf_delete(sb);
1613                 ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1614                 snprintf(ci->error_str, sizeof(ci->error_str),
1615                     "Out of space, %d bytes is too small", cilp->alloc_len);
1616                 return;
1617         }
1618         sbuf_finish(sb);
1619
1620         error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1621         cilp->fill_len = sbuf_len(sb) + 1;
1622         ci->status = CTL_ISCSI_OK;
1623         sbuf_delete(sb);
1624 }
1625
1626 static void
1627 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1628 {
1629         struct icl_pdu *response;
1630         struct iscsi_bhs_asynchronous_message *bhsam;
1631         struct ctl_iscsi_terminate_params *citp;
1632         struct cfiscsi_session *cs;
1633         struct cfiscsi_softc *softc;
1634         int found = 0;
1635
1636         citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1637         softc = &cfiscsi_softc;
1638
1639         mtx_lock(&softc->lock);
1640         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1641                 if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1642                     strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1643                     strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1644                         continue;
1645
1646                 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1647                 if (response == NULL) {
1648                         /*
1649                          * Oh well.  Just terminate the connection.
1650                          */
1651                 } else {
1652                         bhsam = (struct iscsi_bhs_asynchronous_message *)
1653                             response->ip_bhs;
1654                         bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1655                         bhsam->bhsam_flags = 0x80;
1656                         bhsam->bhsam_0xffffffff = 0xffffffff;
1657                         bhsam->bhsam_async_event =
1658                             BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1659                         cfiscsi_pdu_queue(response);
1660                 }
1661                 cfiscsi_session_terminate(cs);
1662                 found++;
1663         }
1664         mtx_unlock(&softc->lock);
1665
1666         if (found == 0) {
1667                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1668                 snprintf(ci->error_str, sizeof(ci->error_str),
1669                     "No matching connections found");
1670                 return;
1671         }
1672
1673         ci->status = CTL_ISCSI_OK;
1674 }
1675
1676 static void
1677 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1678 {
1679         struct icl_pdu *response;
1680         struct iscsi_bhs_asynchronous_message *bhsam;
1681         struct ctl_iscsi_logout_params *cilp;
1682         struct cfiscsi_session *cs;
1683         struct cfiscsi_softc *softc;
1684         int found = 0;
1685
1686         cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1687         softc = &cfiscsi_softc;
1688
1689         mtx_lock(&softc->lock);
1690         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1691                 if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1692                     strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1693                     strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1694                         continue;
1695
1696                 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1697                 if (response == NULL) {
1698                         ci->status = CTL_ISCSI_ERROR;
1699                         snprintf(ci->error_str, sizeof(ci->error_str),
1700                             "Unable to allocate memory");
1701                         mtx_unlock(&softc->lock);
1702                         return;
1703                 }
1704                 bhsam =
1705                     (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1706                 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1707                 bhsam->bhsam_flags = 0x80;
1708                 bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1709                 bhsam->bhsam_parameter3 = htons(10);
1710                 cfiscsi_pdu_queue(response);
1711                 found++;
1712         }
1713         mtx_unlock(&softc->lock);
1714
1715         if (found == 0) {
1716                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1717                 snprintf(ci->error_str, sizeof(ci->error_str),
1718                     "No matching connections found");
1719                 return;
1720         }
1721
1722         ci->status = CTL_ISCSI_OK;
1723 }
1724
1725 #ifdef ICL_KERNEL_PROXY
1726 static void
1727 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1728 {
1729         struct ctl_iscsi_listen_params *cilp;
1730         struct sockaddr *sa;
1731         int error;
1732
1733         cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1734
1735         if (cfiscsi_softc.listener == NULL) {
1736                 CFISCSI_DEBUG("no listener");
1737                 snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1738                 ci->status = CTL_ISCSI_ERROR;
1739                 return;
1740         }
1741
1742         error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1743         if (error != 0) {
1744                 CFISCSI_DEBUG("getsockaddr, error %d", error);
1745                 snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1746                 ci->status = CTL_ISCSI_ERROR;
1747                 return;
1748         }
1749
1750         error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1751             cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1752         if (error != 0) {
1753                 free(sa, M_SONAME);
1754                 CFISCSI_DEBUG("icl_listen_add, error %d", error);
1755                 snprintf(ci->error_str, sizeof(ci->error_str),
1756                     "icl_listen_add failed, error %d", error);
1757                 ci->status = CTL_ISCSI_ERROR;
1758                 return;
1759         }
1760
1761         ci->status = CTL_ISCSI_OK;
1762 }
1763
1764 static void
1765 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1766 {
1767         struct ctl_iscsi_accept_params *ciap;
1768         struct cfiscsi_session *cs;
1769         int error;
1770
1771         ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1772
1773         mtx_lock(&cfiscsi_softc.lock);
1774         for (;;) {
1775                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1776                         if (cs->cs_waiting_for_ctld)
1777                                 break;
1778                 }
1779                 if (cs != NULL)
1780                         break;
1781                 error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1782                 if (error != 0) {
1783                         mtx_unlock(&cfiscsi_softc.lock);
1784                         snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1785                         ci->status = CTL_ISCSI_ERROR;
1786                         return;
1787                 }
1788         }
1789         mtx_unlock(&cfiscsi_softc.lock);
1790
1791         cs->cs_waiting_for_ctld = false;
1792         cs->cs_login_phase = true;
1793
1794         ciap->connection_id = cs->cs_id;
1795         ciap->portal_id = cs->cs_portal_id;
1796         ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1797         error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1798             cs->cs_initiator_sa->sa_len);
1799         if (error != 0) {
1800                 snprintf(ci->error_str, sizeof(ci->error_str),
1801                     "copyout failed with error %d", error);
1802                 ci->status = CTL_ISCSI_ERROR;
1803                 return;
1804         }
1805
1806         ci->status = CTL_ISCSI_OK;
1807 }
1808
1809 static void
1810 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1811 {
1812         struct ctl_iscsi_send_params *cisp;
1813         struct cfiscsi_session *cs;
1814         struct icl_pdu *ip;
1815         size_t datalen;
1816         void *data;
1817         int error;
1818
1819         cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1820
1821         mtx_lock(&cfiscsi_softc.lock);
1822         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1823                 if (cs->cs_id == cisp->connection_id)
1824                         break;
1825         }
1826         if (cs == NULL) {
1827                 mtx_unlock(&cfiscsi_softc.lock);
1828                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1829                 ci->status = CTL_ISCSI_ERROR;
1830                 return;
1831         }
1832         mtx_unlock(&cfiscsi_softc.lock);
1833
1834 #if 0
1835         if (cs->cs_login_phase == false)
1836                 return (EBUSY);
1837 #endif
1838
1839         if (cs->cs_terminating) {
1840                 snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1841                 ci->status = CTL_ISCSI_ERROR;
1842                 return;
1843         }
1844
1845         datalen = cisp->data_segment_len;
1846         /*
1847          * XXX
1848          */
1849         //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
1850         if (datalen > 65535) {
1851                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1852                 ci->status = CTL_ISCSI_ERROR;
1853                 return;
1854         }
1855         if (datalen > 0) {
1856                 data = malloc(datalen, M_CFISCSI, M_WAITOK);
1857                 error = copyin(cisp->data_segment, data, datalen);
1858                 if (error != 0) {
1859                         free(data, M_CFISCSI);
1860                         snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
1861                         ci->status = CTL_ISCSI_ERROR;
1862                         return;
1863                 }
1864         }
1865
1866         ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
1867         memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
1868         if (datalen > 0) {
1869                 icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1870                 free(data, M_CFISCSI);
1871         }
1872         CFISCSI_SESSION_LOCK(cs);
1873         icl_pdu_queue(ip);
1874         CFISCSI_SESSION_UNLOCK(cs);
1875         ci->status = CTL_ISCSI_OK;
1876 }
1877
1878 static void
1879 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
1880 {
1881         struct ctl_iscsi_receive_params *cirp;
1882         struct cfiscsi_session *cs;
1883         struct icl_pdu *ip;
1884         void *data;
1885         int error;
1886
1887         cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
1888
1889         mtx_lock(&cfiscsi_softc.lock);
1890         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1891                 if (cs->cs_id == cirp->connection_id)
1892                         break;
1893         }
1894         if (cs == NULL) {
1895                 mtx_unlock(&cfiscsi_softc.lock);
1896                 snprintf(ci->error_str, sizeof(ci->error_str),
1897                     "connection not found");
1898                 ci->status = CTL_ISCSI_ERROR;
1899                 return;
1900         }
1901         mtx_unlock(&cfiscsi_softc.lock);
1902
1903 #if 0
1904         if (is->is_login_phase == false)
1905                 return (EBUSY);
1906 #endif
1907
1908         CFISCSI_SESSION_LOCK(cs);
1909         while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
1910                 error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
1911                 if (error != 0) {
1912                         CFISCSI_SESSION_UNLOCK(cs);
1913                         snprintf(ci->error_str, sizeof(ci->error_str),
1914                             "interrupted by signal");
1915                         ci->status = CTL_ISCSI_ERROR;
1916                         return;
1917                 }
1918         }
1919
1920         if (cs->cs_terminating) {
1921                 CFISCSI_SESSION_UNLOCK(cs);
1922                 snprintf(ci->error_str, sizeof(ci->error_str),
1923                     "connection terminating");
1924                 ci->status = CTL_ISCSI_ERROR;
1925                 return;
1926         }
1927         ip = cs->cs_login_pdu;
1928         cs->cs_login_pdu = NULL;
1929         CFISCSI_SESSION_UNLOCK(cs);
1930
1931         if (ip->ip_data_len > cirp->data_segment_len) {
1932                 icl_pdu_free(ip);
1933                 snprintf(ci->error_str, sizeof(ci->error_str),
1934                     "data segment too big");
1935                 ci->status = CTL_ISCSI_ERROR;
1936                 return;
1937         }
1938
1939         copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
1940         if (ip->ip_data_len > 0) {
1941                 data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
1942                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1943                 copyout(data, cirp->data_segment, ip->ip_data_len);
1944                 free(data, M_CFISCSI);
1945         }
1946
1947         icl_pdu_free(ip);
1948         ci->status = CTL_ISCSI_OK;
1949 }
1950
1951 #endif /* !ICL_KERNEL_PROXY */
1952
1953 static void
1954 cfiscsi_ioctl_port_create(struct ctl_req *req)
1955 {
1956         struct cfiscsi_target *ct;
1957         struct ctl_port *port;
1958         const char *target, *alias, *tag;
1959         struct scsi_vpd_id_descriptor *desc;
1960         ctl_options_t opts;
1961         int retval, len, idlen;
1962
1963         ctl_init_opts(&opts, req->num_args, req->kern_args);
1964         target = ctl_get_opt(&opts, "cfiscsi_target");
1965         alias = ctl_get_opt(&opts, "cfiscsi_target_alias");
1966         tag = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
1967         if (target == NULL || tag == NULL) {
1968                 req->status = CTL_LUN_ERROR;
1969                 snprintf(req->error_str, sizeof(req->error_str),
1970                     "Missing required argument");
1971                 ctl_free_opts(&opts);
1972                 return;
1973         }
1974         ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias);
1975         if (ct == NULL) {
1976                 req->status = CTL_LUN_ERROR;
1977                 snprintf(req->error_str, sizeof(req->error_str),
1978                     "failed to create target \"%s\"", target);
1979                 ctl_free_opts(&opts);
1980                 return;
1981         }
1982         if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
1983                 req->status = CTL_LUN_ERROR;
1984                 snprintf(req->error_str, sizeof(req->error_str),
1985                     "target \"%s\" already exists", target);
1986                 cfiscsi_target_release(ct);
1987                 ctl_free_opts(&opts);
1988                 return;
1989         }
1990         port = &ct->ct_port;
1991         if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
1992                 goto done;
1993
1994         port->frontend = &cfiscsi_frontend;
1995         port->port_type = CTL_PORT_ISCSI;
1996         /* XXX KDM what should the real number be here? */
1997         port->num_requested_ctl_io = 4096;
1998         port->port_name = "iscsi";
1999         port->physical_port = strtoul(tag, NULL, 0);
2000         port->virtual_port = ct->ct_target_id;
2001         port->port_online = cfiscsi_online;
2002         port->port_offline = cfiscsi_offline;
2003         port->port_info = cfiscsi_info;
2004         port->onoff_arg = ct;
2005         port->lun_enable = cfiscsi_lun_enable;
2006         port->lun_disable = cfiscsi_lun_disable;
2007         port->lun_map = cfiscsi_lun_map;
2008         port->targ_lun_arg = ct;
2009         port->fe_datamove = cfiscsi_datamove;
2010         port->fe_done = cfiscsi_done;
2011
2012         /* XXX KDM what should we report here? */
2013         /* XXX These should probably be fetched from CTL. */
2014         port->max_targets = 1;
2015         port->max_target_id = 15;
2016
2017         port->options = opts;
2018         STAILQ_INIT(&opts);
2019
2020         /* Generate Port ID. */
2021         idlen = strlen(target) + strlen(",t,0x0001") + 1;
2022         idlen = roundup2(idlen, 4);
2023         len = sizeof(struct scsi_vpd_device_id) + idlen;
2024         port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2025             M_CTL, M_WAITOK | M_ZERO);
2026         port->port_devid->len = len;
2027         desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2028         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2029         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2030             SVPD_ID_TYPE_SCSI_NAME;
2031         desc->length = idlen;
2032         snprintf(desc->identifier, idlen, "%s,t,0x%4.4x",
2033             target, port->physical_port);
2034
2035         /* Generate Target ID. */
2036         idlen = strlen(target) + 1;
2037         idlen = roundup2(idlen, 4);
2038         len = sizeof(struct scsi_vpd_device_id) + idlen;
2039         port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2040             M_CTL, M_WAITOK | M_ZERO);
2041         port->target_devid->len = len;
2042         desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2043         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2044         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2045             SVPD_ID_TYPE_SCSI_NAME;
2046         desc->length = idlen;
2047         strlcpy(desc->identifier, target, idlen);
2048
2049         retval = ctl_port_register(port);
2050         if (retval != 0) {
2051                 ctl_free_opts(&port->options);
2052                 cfiscsi_target_release(ct);
2053                 free(port->port_devid, M_CFISCSI);
2054                 free(port->target_devid, M_CFISCSI);
2055                 req->status = CTL_LUN_ERROR;
2056                 snprintf(req->error_str, sizeof(req->error_str),
2057                     "ctl_frontend_register() failed with error %d", retval);
2058                 return;
2059         }
2060 done:
2061         ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2062         req->status = CTL_LUN_OK;
2063         memcpy(req->kern_args[0].kvalue, &port->targ_port,
2064             sizeof(port->targ_port)); //XXX
2065 }
2066
2067 static void
2068 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2069 {
2070         struct cfiscsi_target *ct;
2071         const char *target;
2072         ctl_options_t opts;
2073
2074         ctl_init_opts(&opts, req->num_args, req->kern_args);
2075         target = ctl_get_opt(&opts, "cfiscsi_target");
2076         if (target == NULL) {
2077                 ctl_free_opts(&opts);
2078                 req->status = CTL_LUN_ERROR;
2079                 snprintf(req->error_str, sizeof(req->error_str),
2080                     "Missing required argument");
2081                 return;
2082         }
2083         ct = cfiscsi_target_find(&cfiscsi_softc, target);
2084         if (ct == NULL) {
2085                 ctl_free_opts(&opts);
2086                 req->status = CTL_LUN_ERROR;
2087                 snprintf(req->error_str, sizeof(req->error_str),
2088                     "can't find target \"%s\"", target);
2089                 return;
2090         }
2091         if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) {
2092                 ctl_free_opts(&opts);
2093                 req->status = CTL_LUN_ERROR;
2094                 snprintf(req->error_str, sizeof(req->error_str),
2095                     "target \"%s\" is already dying", target);
2096                 return;
2097         }
2098         ctl_free_opts(&opts);
2099
2100         ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2101         ctl_port_offline(&ct->ct_port);
2102         cfiscsi_target_release(ct);
2103         cfiscsi_target_release(ct);
2104 }
2105
2106 static int
2107 cfiscsi_ioctl(struct cdev *dev,
2108     u_long cmd, caddr_t addr, int flag, struct thread *td)
2109 {
2110         struct ctl_iscsi *ci;
2111         struct ctl_req *req;
2112
2113         if (cmd == CTL_PORT_REQ) {
2114                 req = (struct ctl_req *)addr;
2115                 switch (req->reqtype) {
2116                 case CTL_REQ_CREATE:
2117                         cfiscsi_ioctl_port_create(req);
2118                         break;
2119                 case CTL_REQ_REMOVE:
2120                         cfiscsi_ioctl_port_remove(req);
2121                         break;
2122                 default:
2123                         req->status = CTL_LUN_ERROR;
2124                         snprintf(req->error_str, sizeof(req->error_str),
2125                             "Unsupported request type %d", req->reqtype);
2126                 }
2127                 return (0);
2128         }
2129
2130         if (cmd != CTL_ISCSI)
2131                 return (ENOTTY);
2132
2133         ci = (struct ctl_iscsi *)addr;
2134         switch (ci->type) {
2135         case CTL_ISCSI_HANDOFF:
2136                 cfiscsi_ioctl_handoff(ci);
2137                 break;
2138         case CTL_ISCSI_LIST:
2139                 cfiscsi_ioctl_list(ci);
2140                 break;
2141         case CTL_ISCSI_TERMINATE:
2142                 cfiscsi_ioctl_terminate(ci);
2143                 break;
2144         case CTL_ISCSI_LOGOUT:
2145                 cfiscsi_ioctl_logout(ci);
2146                 break;
2147 #ifdef ICL_KERNEL_PROXY
2148         case CTL_ISCSI_LISTEN:
2149                 cfiscsi_ioctl_listen(ci);
2150                 break;
2151         case CTL_ISCSI_ACCEPT:
2152                 cfiscsi_ioctl_accept(ci);
2153                 break;
2154         case CTL_ISCSI_SEND:
2155                 cfiscsi_ioctl_send(ci);
2156                 break;
2157         case CTL_ISCSI_RECEIVE:
2158                 cfiscsi_ioctl_receive(ci);
2159                 break;
2160 #else
2161         case CTL_ISCSI_LISTEN:
2162         case CTL_ISCSI_ACCEPT:
2163         case CTL_ISCSI_SEND:
2164         case CTL_ISCSI_RECEIVE:
2165                 ci->status = CTL_ISCSI_ERROR;
2166                 snprintf(ci->error_str, sizeof(ci->error_str),
2167                     "%s: CTL compiled without ICL_KERNEL_PROXY",
2168                     __func__);
2169                 break;
2170 #endif /* !ICL_KERNEL_PROXY */
2171         default:
2172                 ci->status = CTL_ISCSI_ERROR;
2173                 snprintf(ci->error_str, sizeof(ci->error_str),
2174                     "%s: invalid iSCSI request type %d", __func__, ci->type);
2175                 break;
2176         }
2177
2178         return (0);
2179 }
2180
2181 static void
2182 cfiscsi_target_hold(struct cfiscsi_target *ct)
2183 {
2184
2185         refcount_acquire(&ct->ct_refcount);
2186 }
2187
2188 static void
2189 cfiscsi_target_release(struct cfiscsi_target *ct)
2190 {
2191         struct cfiscsi_softc *softc;
2192
2193         softc = ct->ct_softc;
2194         mtx_lock(&softc->lock);
2195         if (refcount_release(&ct->ct_refcount)) {
2196                 TAILQ_REMOVE(&softc->targets, ct, ct_next);
2197                 mtx_unlock(&softc->lock);
2198                 if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2199                         ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2200                         if (ctl_port_deregister(&ct->ct_port) != 0)
2201                                 printf("%s: ctl_port_deregister() failed\n",
2202                                     __func__);
2203                 }
2204                 free(ct, M_CFISCSI);
2205
2206                 return;
2207         }
2208         mtx_unlock(&softc->lock);
2209 }
2210
2211 static struct cfiscsi_target *
2212 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name)
2213 {
2214         struct cfiscsi_target *ct;
2215
2216         mtx_lock(&softc->lock);
2217         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2218                 if (strcmp(name, ct->ct_name) != 0 ||
2219                     ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2220                         continue;
2221                 cfiscsi_target_hold(ct);
2222                 mtx_unlock(&softc->lock);
2223                 return (ct);
2224         }
2225         mtx_unlock(&softc->lock);
2226
2227         return (NULL);
2228 }
2229
2230 static struct cfiscsi_target *
2231 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2232     const char *alias)
2233 {
2234         struct cfiscsi_target *ct, *newct;
2235         int i;
2236
2237         if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2238                 return (NULL);
2239
2240         newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2241
2242         mtx_lock(&softc->lock);
2243         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2244                 if (strcmp(name, ct->ct_name) != 0 ||
2245                     ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2246                         continue;
2247                 cfiscsi_target_hold(ct);
2248                 mtx_unlock(&softc->lock);
2249                 free(newct, M_CFISCSI);
2250                 return (ct);
2251         }
2252
2253         for (i = 0; i < CTL_MAX_LUNS; i++)
2254                 newct->ct_luns[i] = UINT32_MAX;
2255
2256         strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2257         if (alias != NULL)
2258                 strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2259         refcount_init(&newct->ct_refcount, 1);
2260         newct->ct_softc = softc;
2261         if (TAILQ_EMPTY(&softc->targets))
2262                 softc->last_target_id = 0;
2263         newct->ct_target_id = ++softc->last_target_id;
2264         TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2265         mtx_unlock(&softc->lock);
2266
2267         return (newct);
2268 }
2269
2270 /*
2271  * Takes LUN from the target space and returns LUN from the CTL space.
2272  */
2273 static uint32_t
2274 cfiscsi_lun_map(void *arg, uint32_t lun)
2275 {
2276         struct cfiscsi_target *ct = arg;
2277
2278         if (lun >= CTL_MAX_LUNS) {
2279                 CFISCSI_DEBUG("requested lun number %d is higher "
2280                     "than maximum %d", lun, CTL_MAX_LUNS - 1);
2281                 return (UINT32_MAX);
2282         }
2283         return (ct->ct_luns[lun]);
2284 }
2285
2286 static int
2287 cfiscsi_target_set_lun(struct cfiscsi_target *ct,
2288     unsigned long lun_id, unsigned long ctl_lun_id)
2289 {
2290
2291         if (lun_id >= CTL_MAX_LUNS) {
2292                 CFISCSI_WARN("requested lun number %ld is higher "
2293                     "than maximum %d", lun_id, CTL_MAX_LUNS - 1);
2294                 return (-1);
2295         }
2296
2297         if (ct->ct_luns[lun_id] < CTL_MAX_LUNS) {
2298                 /*
2299                  * CTL calls cfiscsi_lun_enable() twice for each LUN - once
2300                  * when the LUN is created, and a second time just before
2301                  * the port is brought online; don't emit warnings
2302                  * for that case.
2303                  */
2304                 if (ct->ct_luns[lun_id] == ctl_lun_id)
2305                         return (0);
2306                 CFISCSI_WARN("lun %ld already allocated", lun_id);
2307                 return (-1);
2308         }
2309
2310 #if 0
2311         CFISCSI_DEBUG("adding mapping for lun %ld, target %s "
2312             "to ctl lun %ld", lun_id, ct->ct_name, ctl_lun_id);
2313 #endif
2314
2315         ct->ct_luns[lun_id] = ctl_lun_id;
2316
2317         return (0);
2318 }
2319
2320 static int
2321 cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id)
2322 {
2323         struct cfiscsi_softc *softc;
2324         struct cfiscsi_target *ct;
2325         const char *target = NULL;
2326         const char *lun = NULL;
2327         unsigned long tmp;
2328
2329         ct = (struct cfiscsi_target *)arg;
2330         softc = ct->ct_softc;
2331
2332         target = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options,
2333             "cfiscsi_target");
2334         lun = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options,
2335             "cfiscsi_lun");
2336
2337         if (target == NULL && lun == NULL)
2338                 return (0);
2339
2340         if (target == NULL || lun == NULL) {
2341                 CFISCSI_WARN("lun added with cfiscsi_target, but without "
2342                     "cfiscsi_lun, or the other way around; ignoring");
2343                 return (0);
2344         }
2345
2346         if (strcmp(target, ct->ct_name) != 0)
2347                 return (0);
2348
2349         tmp = strtoul(lun, NULL, 10);
2350         cfiscsi_target_set_lun(ct, tmp, lun_id);
2351         return (0);
2352 }
2353
2354 static int
2355 cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
2356 {
2357         struct cfiscsi_softc *softc;
2358         struct cfiscsi_target *ct;
2359         int i;
2360
2361         ct = (struct cfiscsi_target *)arg;
2362         softc = ct->ct_softc;
2363
2364         mtx_lock(&softc->lock);
2365         for (i = 0; i < CTL_MAX_LUNS; i++) {
2366                 if (ct->ct_luns[i] != lun_id)
2367                         continue;
2368                 ct->ct_luns[i] = UINT32_MAX;
2369                 break;
2370         }
2371         mtx_unlock(&softc->lock);
2372         return (0);
2373 }
2374
2375 static void
2376 cfiscsi_datamove_in(union ctl_io *io)
2377 {
2378         struct cfiscsi_session *cs;
2379         struct icl_pdu *request, *response;
2380         const struct iscsi_bhs_scsi_command *bhssc;
2381         struct iscsi_bhs_data_in *bhsdi;
2382         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2383         size_t len, expected_len, sg_len, buffer_offset;
2384         const char *sg_addr;
2385         int ctl_sg_count, error, i;
2386
2387         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2388         cs = PDU_SESSION(request);
2389
2390         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2391         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2392             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2393             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2394
2395         if (io->scsiio.kern_sg_entries > 0) {
2396                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2397                 ctl_sg_count = io->scsiio.kern_sg_entries;
2398         } else {
2399                 ctl_sglist = &ctl_sg_entry;
2400                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2401                 ctl_sglist->len = io->scsiio.kern_data_len;
2402                 ctl_sg_count = 1;
2403         }
2404
2405         /*
2406          * This is the total amount of data to be transferred within the current
2407          * SCSI command.  We need to record it so that we can properly report
2408          * underflow/underflow.
2409          */
2410         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2411
2412         /*
2413          * This is the offset within the current SCSI command; for the first
2414          * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2415          * it will be the sum of lengths of previous ones.
2416          */
2417         buffer_offset = io->scsiio.kern_rel_offset;
2418
2419         /*
2420          * This is the transfer length expected by the initiator.  In theory,
2421          * it could be different from the correct amount of data from the SCSI
2422          * point of view, even if that doesn't make any sense.
2423          */
2424         expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2425 #if 0
2426         if (expected_len != io->scsiio.kern_total_len) {
2427                 CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, "
2428                     "actual length %zd", expected_len,
2429                     (size_t)io->scsiio.kern_total_len);
2430         }
2431 #endif
2432
2433         if (buffer_offset >= expected_len) {
2434 #if 0
2435                 CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2436                     "already sent the expected len", buffer_offset);
2437 #endif
2438                 io->scsiio.be_move_done(io);
2439                 return;
2440         }
2441
2442         i = 0;
2443         sg_addr = NULL;
2444         sg_len = 0;
2445         response = NULL;
2446         bhsdi = NULL;
2447         for (;;) {
2448                 if (response == NULL) {
2449                         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2450                         if (response == NULL) {
2451                                 CFISCSI_SESSION_WARN(cs, "failed to "
2452                                     "allocate memory; dropping connection");
2453                                 ctl_set_busy(&io->scsiio);
2454                                 io->scsiio.be_move_done(io);
2455                                 cfiscsi_session_terminate(cs);
2456                                 return;
2457                         }
2458                         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2459                         bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2460                         bhsdi->bhsdi_initiator_task_tag =
2461                             bhssc->bhssc_initiator_task_tag;
2462                         bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2463                         PDU_EXPDATASN(request)++;
2464                         bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2465                 }
2466
2467                 KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2468                 if (sg_len == 0) {
2469                         sg_addr = ctl_sglist[i].addr;
2470                         sg_len = ctl_sglist[i].len;
2471                         KASSERT(sg_len > 0, ("sg_len <= 0"));
2472                 }
2473
2474                 len = sg_len;
2475
2476                 /*
2477                  * Truncate to maximum data segment length.
2478                  */
2479                 KASSERT(response->ip_data_len < cs->cs_max_data_segment_length,
2480                     ("ip_data_len %zd >= max_data_segment_length %zd",
2481                     response->ip_data_len, cs->cs_max_data_segment_length));
2482                 if (response->ip_data_len + len >
2483                     cs->cs_max_data_segment_length) {
2484                         len = cs->cs_max_data_segment_length -
2485                             response->ip_data_len;
2486                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2487                             len, sg_len));
2488                 }
2489
2490                 /*
2491                  * Truncate to expected data transfer length.
2492                  */
2493                 KASSERT(buffer_offset + response->ip_data_len < expected_len,
2494                     ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2495                     buffer_offset, response->ip_data_len, expected_len));
2496                 if (buffer_offset + response->ip_data_len + len > expected_len) {
2497                         CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2498                             "to expected data transfer length %zd",
2499                             buffer_offset + response->ip_data_len + len, expected_len);
2500                         len = expected_len - (buffer_offset + response->ip_data_len);
2501                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2502                             len, sg_len));
2503                 }
2504
2505                 error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT);
2506                 if (error != 0) {
2507                         CFISCSI_SESSION_WARN(cs, "failed to "
2508                             "allocate memory; dropping connection");
2509                         icl_pdu_free(response);
2510                         ctl_set_busy(&io->scsiio);
2511                         io->scsiio.be_move_done(io);
2512                         cfiscsi_session_terminate(cs);
2513                         return;
2514                 }
2515                 sg_addr += len;
2516                 sg_len -= len;
2517
2518                 KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2519                     ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2520                     buffer_offset, response->ip_data_len, expected_len));
2521                 if (buffer_offset + response->ip_data_len == expected_len) {
2522                         /*
2523                          * Already have the amount of data the initiator wanted.
2524                          */
2525                         break;
2526                 }
2527
2528                 if (sg_len == 0) {
2529                         /*
2530                          * End of scatter-gather segment;
2531                          * proceed to the next one...
2532                          */
2533                         if (i == ctl_sg_count - 1) {
2534                                 /*
2535                                  * ... unless this was the last one.
2536                                  */
2537                                 break;
2538                         }
2539                         i++;
2540                 }
2541
2542                 if (response->ip_data_len == cs->cs_max_data_segment_length) {
2543                         /*
2544                          * Can't stuff more data into the current PDU;
2545                          * queue it.  Note that's not enough to check
2546                          * for kern_data_resid == 0 instead; there
2547                          * may be several Data-In PDUs for the final
2548                          * call to cfiscsi_datamove(), and we want
2549                          * to set the F flag only on the last of them.
2550                          */
2551                         buffer_offset += response->ip_data_len;
2552                         if (buffer_offset == io->scsiio.kern_total_len ||
2553                             buffer_offset == expected_len) {
2554                                 buffer_offset -= response->ip_data_len;
2555                                 break;
2556                         }
2557                         cfiscsi_pdu_queue(response);
2558                         response = NULL;
2559                         bhsdi = NULL;
2560                 }
2561         }
2562         if (response != NULL) {
2563                 buffer_offset += response->ip_data_len;
2564                 if (buffer_offset == io->scsiio.kern_total_len ||
2565                     buffer_offset == expected_len) {
2566                         bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2567                         if (io->io_hdr.status == CTL_SUCCESS) {
2568                                 bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2569                                 if (PDU_TOTAL_TRANSFER_LEN(request) <
2570                                     ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2571                                         bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2572                                         bhsdi->bhsdi_residual_count =
2573                                             htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2574                                             PDU_TOTAL_TRANSFER_LEN(request));
2575                                 } else if (PDU_TOTAL_TRANSFER_LEN(request) >
2576                                     ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2577                                         bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2578                                         bhsdi->bhsdi_residual_count =
2579                                             htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2580                                             ntohl(bhssc->bhssc_expected_data_transfer_length));
2581                                 }
2582                                 bhsdi->bhsdi_status = io->scsiio.scsi_status;
2583                                 io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2584                         }
2585                 }
2586                 KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2587                 cfiscsi_pdu_queue(response);
2588         }
2589
2590         io->scsiio.be_move_done(io);
2591 }
2592
2593 static void
2594 cfiscsi_datamove_out(union ctl_io *io)
2595 {
2596         struct cfiscsi_session *cs;
2597         struct icl_pdu *request, *response;
2598         const struct iscsi_bhs_scsi_command *bhssc;
2599         struct iscsi_bhs_r2t *bhsr2t;
2600         struct cfiscsi_data_wait *cdw;
2601         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2602         uint32_t expected_len, r2t_off, r2t_len;
2603         uint32_t target_transfer_tag;
2604         bool done;
2605
2606         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2607         cs = PDU_SESSION(request);
2608
2609         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2610         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2611             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2612             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2613
2614         /*
2615          * We need to record it so that we can properly report
2616          * underflow/underflow.
2617          */
2618         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2619
2620         /*
2621          * Report write underflow as error since CTL and backends don't
2622          * really support it, and SCSI does not tell how to do it right.
2623          */
2624         expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2625         if (io->scsiio.kern_rel_offset + io->scsiio.kern_data_len >
2626             expected_len) {
2627                 io->scsiio.io_hdr.port_status = 43;
2628                 io->scsiio.be_move_done(io);
2629                 return;
2630         }
2631
2632         target_transfer_tag =
2633             atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2634
2635 #if 0
2636         CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2637             "task tag 0x%x, target transfer tag 0x%x",
2638             bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2639 #endif
2640         cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
2641         if (cdw == NULL) {
2642                 CFISCSI_SESSION_WARN(cs, "failed to "
2643                     "allocate memory; dropping connection");
2644                 ctl_set_busy(&io->scsiio);
2645                 io->scsiio.be_move_done(io);
2646                 cfiscsi_session_terminate(cs);
2647                 return;
2648         }
2649         cdw->cdw_ctl_io = io;
2650         cdw->cdw_target_transfer_tag = target_transfer_tag;
2651         cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2652         cdw->cdw_r2t_end = io->scsiio.kern_data_len;
2653
2654         /* Set initial data pointer for the CDW respecting ext_data_filled. */
2655         if (io->scsiio.kern_sg_entries > 0) {
2656                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2657         } else {
2658                 ctl_sglist = &ctl_sg_entry;
2659                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2660                 ctl_sglist->len = io->scsiio.kern_data_len;
2661         }
2662         cdw->cdw_sg_index = 0;
2663         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2664         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2665         r2t_off = io->scsiio.ext_data_filled;
2666         while (r2t_off > 0) {
2667                 if (r2t_off >= cdw->cdw_sg_len) {
2668                         r2t_off -= cdw->cdw_sg_len;
2669                         cdw->cdw_sg_index++;
2670                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2671                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2672                         continue;
2673                 }
2674                 cdw->cdw_sg_addr += r2t_off;
2675                 cdw->cdw_sg_len -= r2t_off;
2676                 r2t_off = 0;
2677         }
2678
2679         if (cs->cs_immediate_data &&
2680             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2681             icl_pdu_data_segment_length(request)) {
2682                 done = cfiscsi_handle_data_segment(request, cdw);
2683                 if (done) {
2684                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2685                         io->scsiio.be_move_done(io);
2686                         return;
2687                 }
2688         }
2689
2690         r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2691         r2t_len = MIN(io->scsiio.kern_data_len - io->scsiio.ext_data_filled,
2692             cs->cs_max_burst_length);
2693         cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2694
2695         CFISCSI_SESSION_LOCK(cs);
2696         TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2697         CFISCSI_SESSION_UNLOCK(cs);
2698
2699         /*
2700          * XXX: We should limit the number of outstanding R2T PDUs
2701          *      per task to MaxOutstandingR2T.
2702          */
2703         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2704         if (response == NULL) {
2705                 CFISCSI_SESSION_WARN(cs, "failed to "
2706                     "allocate memory; dropping connection");
2707                 ctl_set_busy(&io->scsiio);
2708                 io->scsiio.be_move_done(io);
2709                 cfiscsi_session_terminate(cs);
2710                 return;
2711         }
2712         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2713         bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2714         bhsr2t->bhsr2t_flags = 0x80;
2715         bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2716         bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2717         bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2718         /*
2719          * XXX: Here we assume that cfiscsi_datamove() won't ever
2720          *      be running concurrently on several CPUs for a given
2721          *      command.
2722          */
2723         bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2724         PDU_R2TSN(request)++;
2725         /*
2726          * This is the offset within the current SCSI command;
2727          * i.e. for the first call of datamove(), it will be 0,
2728          * and for subsequent ones it will be the sum of lengths
2729          * of previous ones.
2730          *
2731          * The ext_data_filled is to account for unsolicited
2732          * (immediate) data that might have already arrived.
2733          */
2734         bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2735         /*
2736          * This is the total length (sum of S/G lengths) this call
2737          * to cfiscsi_datamove() is supposed to handle, limited by
2738          * MaxBurstLength.
2739          */
2740         bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2741         cfiscsi_pdu_queue(response);
2742 }
2743
2744 static void
2745 cfiscsi_datamove(union ctl_io *io)
2746 {
2747
2748         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2749                 cfiscsi_datamove_in(io);
2750         else {
2751                 /* We hadn't received anything during this datamove yet. */
2752                 io->scsiio.ext_data_filled = 0;
2753                 cfiscsi_datamove_out(io);
2754         }
2755 }
2756
2757 static void
2758 cfiscsi_scsi_command_done(union ctl_io *io)
2759 {
2760         struct icl_pdu *request, *response;
2761         struct iscsi_bhs_scsi_command *bhssc;
2762         struct iscsi_bhs_scsi_response *bhssr;
2763 #ifdef DIAGNOSTIC
2764         struct cfiscsi_data_wait *cdw;
2765 #endif
2766         struct cfiscsi_session *cs;
2767         uint16_t sense_length;
2768
2769         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2770         cs = PDU_SESSION(request);
2771         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2772         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2773             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2774             ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2775
2776         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2777         //    bhssc->bhssc_initiator_task_tag);
2778
2779 #ifdef DIAGNOSTIC
2780         CFISCSI_SESSION_LOCK(cs);
2781         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2782                 KASSERT(bhssc->bhssc_initiator_task_tag !=
2783                     cdw->cdw_initiator_task_tag, ("dangling cdw"));
2784         CFISCSI_SESSION_UNLOCK(cs);
2785 #endif
2786
2787         /*
2788          * Do not return status for aborted commands.
2789          * There are exceptions, but none supported by CTL yet.
2790          */
2791         if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2792              (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2793             (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2794                 ctl_free_io(io);
2795                 icl_pdu_free(request);
2796                 return;
2797         }
2798
2799         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2800         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2801         bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2802         bhssr->bhssr_flags = 0x80;
2803         /*
2804          * XXX: We don't deal with bidirectional under/overflows;
2805          *      does anything actually support those?
2806          */
2807         if (PDU_TOTAL_TRANSFER_LEN(request) <
2808             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2809                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2810                 bhssr->bhssr_residual_count =
2811                     htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2812                     PDU_TOTAL_TRANSFER_LEN(request));
2813                 //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2814                 //    ntohl(bhssr->bhssr_residual_count));
2815         } else if (PDU_TOTAL_TRANSFER_LEN(request) >
2816             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2817                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2818                 bhssr->bhssr_residual_count =
2819                     htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2820                     ntohl(bhssc->bhssc_expected_data_transfer_length));
2821                 //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2822                 //    ntohl(bhssr->bhssr_residual_count));
2823         }
2824         bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2825         bhssr->bhssr_status = io->scsiio.scsi_status;
2826         bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2827         bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2828
2829         if (io->scsiio.sense_len > 0) {
2830 #if 0
2831                 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2832                     io->scsiio.sense_len);
2833 #endif
2834                 sense_length = htons(io->scsiio.sense_len);
2835                 icl_pdu_append_data(response,
2836                     &sense_length, sizeof(sense_length), M_WAITOK);
2837                 icl_pdu_append_data(response,
2838                     &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2839         }
2840
2841         ctl_free_io(io);
2842         icl_pdu_free(request);
2843         cfiscsi_pdu_queue(response);
2844 }
2845
2846 static void
2847 cfiscsi_task_management_done(union ctl_io *io)
2848 {
2849         struct icl_pdu *request, *response;
2850         struct iscsi_bhs_task_management_request *bhstmr;
2851         struct iscsi_bhs_task_management_response *bhstmr2;
2852         struct cfiscsi_data_wait *cdw, *tmpcdw;
2853         struct cfiscsi_session *cs;
2854
2855         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2856         cs = PDU_SESSION(request);
2857         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2858         KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2859             ISCSI_BHS_OPCODE_TASK_REQUEST,
2860             ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2861
2862 #if 0
2863         CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2864             bhstmr->bhstmr_initiator_task_tag,
2865             bhstmr->bhstmr_referenced_task_tag);
2866 #endif
2867
2868         if ((bhstmr->bhstmr_function & ~0x80) ==
2869             BHSTMR_FUNCTION_ABORT_TASK) {
2870                 /*
2871                  * Make sure we no longer wait for Data-Out for this command.
2872                  */
2873                 CFISCSI_SESSION_LOCK(cs);
2874                 TAILQ_FOREACH_SAFE(cdw,
2875                     &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2876                         if (bhstmr->bhstmr_referenced_task_tag !=
2877                             cdw->cdw_initiator_task_tag)
2878                                 continue;
2879
2880 #if 0
2881                         CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2882                             "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2883 #endif
2884                         TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2885                             cdw, cdw_next);
2886                         cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2887                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2888                 }
2889                 CFISCSI_SESSION_UNLOCK(cs);
2890         }
2891
2892         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2893         bhstmr2 = (struct iscsi_bhs_task_management_response *)
2894             response->ip_bhs;
2895         bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2896         bhstmr2->bhstmr_flags = 0x80;
2897         if (io->io_hdr.status == CTL_SUCCESS) {
2898                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2899         } else {
2900                 /*
2901                  * XXX: How to figure out what exactly went wrong?  iSCSI spec
2902                  *      expects us to provide detailed error, e.g. "Task does
2903                  *      not exist" or "LUN does not exist".
2904                  */
2905                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED");
2906                 bhstmr2->bhstmr_response =
2907                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2908         }
2909         bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2910
2911         ctl_free_io(io);
2912         icl_pdu_free(request);
2913         cfiscsi_pdu_queue(response);
2914 }
2915
2916 static void
2917 cfiscsi_done(union ctl_io *io)
2918 {
2919         struct icl_pdu *request;
2920         struct cfiscsi_session *cs;
2921
2922         KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2923                 ("invalid CTL status %#x", io->io_hdr.status));
2924
2925         if (io->io_hdr.io_type == CTL_IO_TASK &&
2926             io->taskio.task_action == CTL_TASK_I_T_NEXUS_RESET) {
2927                 /*
2928                  * Implicit task termination has just completed; nothing to do.
2929                  */
2930                 cs = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2931                 cs->cs_tasks_aborted = true;
2932                 refcount_release(&cs->cs_outstanding_ctl_pdus);
2933                 wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
2934                 ctl_free_io(io);
2935                 return;
2936         }
2937
2938         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2939         cs = PDU_SESSION(request);
2940         refcount_release(&cs->cs_outstanding_ctl_pdus);
2941
2942         switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
2943         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
2944                 cfiscsi_scsi_command_done(io);
2945                 break;
2946         case ISCSI_BHS_OPCODE_TASK_REQUEST:
2947                 cfiscsi_task_management_done(io);
2948                 break;
2949         default:
2950                 panic("cfiscsi_done called with wrong opcode 0x%x",
2951                     request->ip_bhs->bhs_opcode);
2952         }
2953 }