sys/cam/ctl/ctl_frontend_iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 /*
  33  * CTL frontend for the iSCSI protocol.
  34  */
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include <sys/param.h>
  40 #include <sys/capability.h>
  41 #include <sys/condvar.h>
  42 #include <sys/file.h>
  43 #include <sys/kernel.h>
  44 #include <sys/kthread.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mutex.h>
  49 #include <sys/queue.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/systm.h>
  53 #include <sys/uio.h>
  54 #include <sys/unistd.h>
  55 #include <vm/uma.h>
  56
  57 #include <cam/scsi/scsi_all.h>
  58 #include <cam/scsi/scsi_da.h>
  59 #include <cam/ctl/ctl_io.h>
  60 #include <cam/ctl/ctl.h>
  61 #include <cam/ctl/ctl_backend.h>
  62 #include <cam/ctl/ctl_error.h>
  63 #include <cam/ctl/ctl_frontend.h>
  64 #include <cam/ctl/ctl_frontend_internal.h>
  65 #include <cam/ctl/ctl_debug.h>
  66 #include <cam/ctl/ctl_ha.h>
  67 #include <cam/ctl/ctl_ioctl.h>
  68 #include <cam/ctl/ctl_private.h>
  69
  70 #include "../../dev/iscsi/icl.h"
  71 #include "../../dev/iscsi/iscsi_proto.h"
  72 #include "ctl_frontend_iscsi.h"
  73
  74 #ifdef ICL_KERNEL_PROXY
  75 #include <sys/socketvar.h>
  76 #endif
  77
  78 #ifdef ICL_KERNEL_PROXY
  79 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
  80 #endif
  81
  82 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
  83 static uma_zone_t cfiscsi_data_wait_zone;
  84
  85 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
  86     "CAM Target Layer iSCSI Frontend");
  87 static int debug = 3;
  88 TUNABLE_INT("kern.cam.ctl.iscsi.debug", &debug);
  89 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
  90     &debug, 1, "Enable debug messages");
  91 static int ping_timeout = 5;
  92 TUNABLE_INT("kern.cam.ctl.iscsi.ping_timeout", &ping_timeout);
  93 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
  94     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
  95 static int login_timeout = 60;
  96 TUNABLE_INT("kern.cam.ctl.iscsi.login_timeout", &login_timeout);
  97 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
  98     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
  99 static int maxcmdsn_delta = 256;
 100 TUNABLE_INT("kern.cam.ctl.iscsi.maxcmdsn_delta", &maxcmdsn_delta);
 101 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RWTUN,
 102     &maxcmdsn_delta, 256, "Number of commands the initiator can send "
 103     "without confirmation");
 104
 105 #define CFISCSI_DEBUG(X, ...)                                           \
 106         do {                                                            \
 107                 if (debug > 1) {                                        \
 108                         printf("%s: " X "\n",                           \
 109                             __func__, ## __VA_ARGS__);                  \
 110                 }                                                       \
 111         } while (0)
 112
 113 #define CFISCSI_WARN(X, ...)                                            \
 114         do {                                                            \
 115                 if (debug > 0) {                                        \
 116                         printf("WARNING: %s: " X "\n",                  \
 117                             __func__, ## __VA_ARGS__);                  \
 118                 }                                                       \
 119         } while (0)
 120
 121 #define CFISCSI_SESSION_DEBUG(S, X, ...)                                \
 122         do {                                                            \
 123                 if (debug > 1) {                                        \
 124                         printf("%s: %s (%s): " X "\n",                  \
 125                             __func__, S->cs_initiator_addr,             \
 126                             S->cs_initiator_name, ## __VA_ARGS__);      \
 127                 }                                                       \
 128         } while (0)
 129
 130 #define CFISCSI_SESSION_WARN(S, X, ...)                                 \
 131         do  {                                                           \
 132                 if (debug > 0) {                                        \
 133                         printf("WARNING: %s (%s): " X "\n",             \
 134                             S->cs_initiator_addr,                       \
 135                             S->cs_initiator_name, ## __VA_ARGS__);      \
 136                 }                                                       \
 137         } while (0)
 138
 139 #define CFISCSI_SESSION_LOCK(X)         mtx_lock(&X->cs_lock)
 140 #define CFISCSI_SESSION_UNLOCK(X)       mtx_unlock(&X->cs_lock)
 141 #define CFISCSI_SESSION_LOCK_ASSERT(X)  mtx_assert(&X->cs_lock, MA_OWNED)
 142
 143 #define CONN_SESSION(X)                 ((struct cfiscsi_session *)(X)->ic_prv0)
 144 #define PDU_SESSION(X)                  CONN_SESSION((X)->ip_conn)
 145 #define PDU_EXPDATASN(X)                (X)->ip_prv0
 146 #define PDU_TOTAL_TRANSFER_LEN(X)       (X)->ip_prv1
 147 #define PDU_R2TSN(X)                    (X)->ip_prv2
 148
 149 int             cfiscsi_init(void);
 150 static void     cfiscsi_online(void *arg);
 151 static void     cfiscsi_offline(void *arg);
 152 static int      cfiscsi_lun_enable(void *arg,
 153                     struct ctl_id target_id, int lun_id);
 154 static int      cfiscsi_lun_disable(void *arg,
 155                     struct ctl_id target_id, int lun_id);
 156 static int      cfiscsi_ioctl(struct cdev *dev,
 157                     u_long cmd, caddr_t addr, int flag, struct thread *td);
 158 static int      cfiscsi_devid(struct ctl_scsiio *ctsio, int alloc_len);
 159 static void     cfiscsi_datamove(union ctl_io *io);
 160 static void     cfiscsi_done(union ctl_io *io);
 161 static uint32_t cfiscsi_map_lun(void *arg, uint32_t lun);
 162 static bool     cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
 163 static void     cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
 164 static void     cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
 165 static void     cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
 166 static void     cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
 167 static void     cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
 168 static void     cfiscsi_session_terminate(struct cfiscsi_session *cs);
 169 static struct cfiscsi_target    *cfiscsi_target_find(struct cfiscsi_softc
 170                     *softc, const char *name);
 171 static struct cfiscsi_target    *cfiscsi_target_find_or_create(
 172     struct cfiscsi_softc *softc, const char *name, const char *alias);
 173 static void     cfiscsi_target_release(struct cfiscsi_target *ct);
 174 static void     cfiscsi_session_delete(struct cfiscsi_session *cs);
 175
 176 static struct cfiscsi_softc cfiscsi_softc;
 177 extern struct ctl_softc *control_softc;
 178
 179 static struct ctl_frontend cfiscsi_frontend =
 180 {
 181         .name = "iscsi",
 182         .init = cfiscsi_init,
 183         .ioctl = cfiscsi_ioctl,
 184 };
 185 CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend);
 186
 187 static struct icl_pdu *
 188 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
 189 {
 190
 191         return (icl_pdu_new_bhs(request->ip_conn, flags));
 192 }
 193
 194 static bool
 195 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
 196 {
 197         const struct iscsi_bhs_scsi_command *bhssc;
 198         struct cfiscsi_session *cs;
 199         uint32_t cmdsn, expstatsn;
 200
 201         cs = PDU_SESSION(request);
 202
 203         /*
 204          * Every incoming PDU - not just NOP-Out - resets the ping timer.
 205          * The purpose of the timeout is to reset the connection when it stalls;
 206          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 207          * in some queue.
 208          *
 209          * XXX: Locking?
 210          */
 211         cs->cs_timeout = 0;
 212
 213         /*
 214          * Data-Out PDUs don't contain CmdSN.
 215          */
 216         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 217             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 218                 return (false);
 219
 220         /*
 221          * We're only using fields common for all the request
 222          * (initiator -> target) PDUs.
 223          */
 224         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
 225         cmdsn = ntohl(bhssc->bhssc_cmdsn);
 226         expstatsn = ntohl(bhssc->bhssc_expstatsn);
 227
 228         CFISCSI_SESSION_LOCK(cs);
 229 #if 0
 230         if (expstatsn != cs->cs_statsn) {
 231                 CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
 232                     "while current StatSN is %d", expstatsn,
 233                     cs->cs_statsn);
 234         }
 235 #endif
 236
 237         /*
 238          * The target MUST silently ignore any non-immediate command outside
 239          * of this range.
 240          */
 241         if (cmdsn < cs->cs_cmdsn || cmdsn > cs->cs_cmdsn + maxcmdsn_delta) {
 242                 CFISCSI_SESSION_UNLOCK(cs);
 243                 CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %d, "
 244                     "while expected CmdSN was %d", cmdsn, cs->cs_cmdsn);
 245                 return (true);
 246         }
 247
 248         if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 249                 cs->cs_cmdsn++;
 250
 251         CFISCSI_SESSION_UNLOCK(cs);
 252
 253         return (false);
 254 }
 255
 256 static void
 257 cfiscsi_pdu_handle(struct icl_pdu *request)
 258 {
 259         struct cfiscsi_session *cs;
 260         bool ignore;
 261
 262         cs = PDU_SESSION(request);
 263
 264         ignore = cfiscsi_pdu_update_cmdsn(request);
 265         if (ignore) {
 266                 icl_pdu_free(request);
 267                 return;
 268         }
 269
 270         /*
 271          * Handle the PDU; this includes e.g. receiving the remaining
 272          * part of PDU and submitting the SCSI command to CTL
 273          * or queueing a reply.  The handling routine is responsible
 274          * for freeing the PDU when it's no longer needed.
 275          */
 276         switch (request->ip_bhs->bhs_opcode &
 277             ~ISCSI_BHS_OPCODE_IMMEDIATE) {
 278         case ISCSI_BHS_OPCODE_NOP_OUT:
 279                 cfiscsi_pdu_handle_nop_out(request);
 280                 break;
 281         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
 282                 cfiscsi_pdu_handle_scsi_command(request);
 283                 break;
 284         case ISCSI_BHS_OPCODE_TASK_REQUEST:
 285                 cfiscsi_pdu_handle_task_request(request);
 286                 break;
 287         case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
 288                 cfiscsi_pdu_handle_data_out(request);
 289                 break;
 290         case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
 291                 cfiscsi_pdu_handle_logout_request(request);
 292                 break;
 293         default:
 294                 CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
 295                     "opcode 0x%x; dropping connection",
 296                     request->ip_bhs->bhs_opcode);
 297                 icl_pdu_free(request);
 298                 cfiscsi_session_terminate(cs);
 299         }
 300
 301 }
 302
 303 static void
 304 cfiscsi_receive_callback(struct icl_pdu *request)
 305 {
 306         struct cfiscsi_session *cs;
 307
 308         cs = PDU_SESSION(request);
 309
 310 #ifdef ICL_KERNEL_PROXY
 311         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 312                 if (cs->cs_login_pdu == NULL)
 313                         cs->cs_login_pdu = request;
 314                 else
 315                         icl_pdu_free(request);
 316                 cv_signal(&cs->cs_login_cv);
 317                 return;
 318         }
 319 #endif
 320
 321         cfiscsi_pdu_handle(request);
 322 }
 323
 324 static void
 325 cfiscsi_error_callback(struct icl_conn *ic)
 326 {
 327         struct cfiscsi_session *cs;
 328
 329         cs = CONN_SESSION(ic);
 330
 331         CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
 332         cfiscsi_session_terminate(cs);
 333 }
 334
 335 static int
 336 cfiscsi_pdu_prepare(struct icl_pdu *response)
 337 {
 338         struct cfiscsi_session *cs;
 339         struct iscsi_bhs_scsi_response *bhssr;
 340         bool advance_statsn = true;
 341
 342         cs = PDU_SESSION(response);
 343
 344         CFISCSI_SESSION_LOCK_ASSERT(cs);
 345
 346         /*
 347          * We're only using fields common for all the response
 348          * (target -> initiator) PDUs.
 349          */
 350         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 351
 352         /*
 353          * 10.8.3: "The StatSN for this connection is not advanced
 354          * after this PDU is sent."
 355          */
 356         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
 357                 advance_statsn = false;
 358
 359         /*
 360          * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
 361          * StatSN for the connection is not advanced after this PDU is sent."
 362          */
 363         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
 364             bhssr->bhssr_initiator_task_tag == 0xffffffff)
 365                 advance_statsn = false;
 366
 367         /*
 368          * See the comment below - StatSN is not meaningful and must
 369          * not be advanced.
 370          */
 371         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN)
 372                 advance_statsn = false;
 373
 374         /*
 375          * 10.7.3: "The fields StatSN, Status, and Residual Count
 376          * only have meaningful content if the S bit is set to 1."
 377          */
 378         if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN)
 379                 bhssr->bhssr_statsn = htonl(cs->cs_statsn);
 380         bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
 381         bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn + maxcmdsn_delta);
 382
 383         if (advance_statsn)
 384                 cs->cs_statsn++;
 385
 386         return (0);
 387 }
 388
 389 static void
 390 cfiscsi_pdu_queue(struct icl_pdu *response)
 391 {
 392         struct cfiscsi_session *cs;
 393
 394         cs = PDU_SESSION(response);
 395
 396         CFISCSI_SESSION_LOCK(cs);
 397         cfiscsi_pdu_prepare(response);
 398         icl_pdu_queue(response);
 399         CFISCSI_SESSION_UNLOCK(cs);
 400 }
 401
 402 static uint32_t
 403 cfiscsi_decode_lun(uint64_t encoded)
 404 {
 405         uint8_t lun[8];
 406         uint32_t result;
 407
 408         /*
 409          * The LUN field in iSCSI PDUs may look like an ordinary 64 bit number,
 410          * but is in fact an evil, multidimensional structure defined
 411          * in SCSI Architecture Model 5 (SAM-5), section 4.6.
 412          */
 413         memcpy(lun, &encoded, sizeof(lun));
 414         switch (lun[0] & 0xC0) {
 415         case 0x00:
 416                 if ((lun[0] & 0x3f) != 0 || lun[2] != 0 || lun[3] != 0 ||
 417                     lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) {
 418                         CFISCSI_WARN("malformed LUN "
 419                             "(peripheral device addressing method): 0x%jx",
 420                             (uintmax_t)encoded);
 421                         result = 0xffffffff;
 422                         break;
 423                 }
 424                 result = lun[1];
 425                 break;
 426         case 0x40:
 427                 if (lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 ||
 428                     lun[6] != 0 || lun[7] != 0) {
 429                         CFISCSI_WARN("malformed LUN "
 430                             "(flat address space addressing method): 0x%jx",
 431                             (uintmax_t)encoded);
 432                         result = 0xffffffff;
 433                         break;
 434                 }
 435                 result = ((lun[0] & 0x3f) << 8) + lun[1];
 436                 break;
 437         case 0xC0:
 438                 if (lun[0] != 0xD2 || lun[4] != 0 || lun[5] != 0 ||
 439                     lun[6] != 0 || lun[7] != 0) {
 440                         CFISCSI_WARN("malformed LUN (extended flat "
 441                             "address space addressing method): 0x%jx",
 442                             (uintmax_t)encoded);
 443                         result = 0xffffffff;
 444                         break;
 445                 }
 446                 result = (lun[1] << 16) + (lun[2] << 8) + lun[3];
 447         default:
 448                 CFISCSI_WARN("unsupported LUN format 0x%jx",
 449                     (uintmax_t)encoded);
 450                 result = 0xffffffff;
 451                 break;
 452         }
 453
 454         return (result);
 455 }
 456
 457 static void
 458 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
 459 {
 460         struct cfiscsi_session *cs;
 461         struct iscsi_bhs_nop_out *bhsno;
 462         struct iscsi_bhs_nop_in *bhsni;
 463         struct icl_pdu *response;
 464         void *data = NULL;
 465         size_t datasize;
 466         int error;
 467
 468         cs = PDU_SESSION(request);
 469         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 470
 471         if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
 472                 /*
 473                  * Nothing to do, iscsi_pdu_update_statsn() already
 474                  * zeroed the timeout.
 475                  */
 476                 icl_pdu_free(request);
 477                 return;
 478         }
 479
 480         datasize = icl_pdu_data_segment_length(request);
 481         if (datasize > 0) {
 482                 data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
 483                 if (data == NULL) {
 484                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 485                             "dropping connection");
 486                         icl_pdu_free(request);
 487                         cfiscsi_session_terminate(cs);
 488                         return;
 489                 }
 490                 icl_pdu_get_data(request, 0, data, datasize);
 491         }
 492
 493         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 494         if (response == NULL) {
 495                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 496                     "droppping connection");
 497                 free(data, M_CFISCSI);
 498                 icl_pdu_free(request);
 499                 cfiscsi_session_terminate(cs);
 500                 return;
 501         }
 502         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 503         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
 504         bhsni->bhsni_flags = 0x80;
 505         bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
 506         bhsni->bhsni_target_transfer_tag = 0xffffffff;
 507         if (datasize > 0) {
 508                 error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
 509                 if (error != 0) {
 510                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 511                             "dropping connection");
 512                         free(data, M_CFISCSI);
 513                         icl_pdu_free(request);
 514                         icl_pdu_free(response);
 515                         cfiscsi_session_terminate(cs);
 516                         return;
 517                 }
 518                 free(data, M_CFISCSI);
 519         }
 520
 521         icl_pdu_free(request);
 522         cfiscsi_pdu_queue(response);
 523 }
 524
 525 static void
 526 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
 527 {
 528         struct iscsi_bhs_scsi_command *bhssc;
 529         struct cfiscsi_session *cs;
 530         union ctl_io *io;
 531         int error;
 532
 533         cs = PDU_SESSION(request);
 534         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 535         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
 536         //    bhssc->bhssc_initiator_task_tag);
 537
 538         if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
 539                 CFISCSI_SESSION_WARN(cs, "unsolicited data with "
 540                     "ImmediateData=No; dropping connection");
 541                 icl_pdu_free(request);
 542                 cfiscsi_session_terminate(cs);
 543                 return;
 544         }
 545         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
 546         if (io == NULL) {
 547                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io; "
 548                     "dropping connection");
 549                 icl_pdu_free(request);
 550                 cfiscsi_session_terminate(cs);
 551                 return;
 552         }
 553         ctl_zero_io(io);
 554         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 555         io->io_hdr.io_type = CTL_IO_SCSI;
 556         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 557         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
 558         io->io_hdr.nexus.targ_target.id = 0;
 559         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhssc->bhssc_lun);
 560         io->io_hdr.nexus.lun_map_fn = cfiscsi_map_lun;
 561         io->io_hdr.nexus.lun_map_arg = cs;
 562         io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
 563         switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
 564         case BHSSC_FLAGS_ATTR_UNTAGGED:
 565                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 566                 break;
 567         case BHSSC_FLAGS_ATTR_SIMPLE:
 568                 io->scsiio.tag_type = CTL_TAG_SIMPLE;
 569                 break;
 570         case BHSSC_FLAGS_ATTR_ORDERED:
 571                 io->scsiio.tag_type = CTL_TAG_ORDERED;
 572                 break;
 573         case BHSSC_FLAGS_ATTR_HOQ:
 574                 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
 575                 break;
 576         case BHSSC_FLAGS_ATTR_ACA:
 577                 io->scsiio.tag_type = CTL_TAG_ACA;
 578                 break;
 579         default:
 580                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 581                 CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
 582                     bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
 583                 break;
 584         }
 585         io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
 586         memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
 587         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 588         error = ctl_queue(io);
 589         if (error != CTL_RETVAL_COMPLETE) {
 590                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 591                     "dropping connection", error);
 592                 ctl_free_io(io);
 593                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 594                 icl_pdu_free(request);
 595                 cfiscsi_session_terminate(cs);
 596         }
 597 }
 598
 599 static void
 600 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
 601 {
 602         struct iscsi_bhs_task_management_request *bhstmr;
 603         struct iscsi_bhs_task_management_response *bhstmr2;
 604         struct icl_pdu *response;
 605         struct cfiscsi_session *cs;
 606         union ctl_io *io;
 607         int error;
 608
 609         cs = PDU_SESSION(request);
 610         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
 611         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
 612         if (io == NULL) {
 613                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io;"
 614                     "dropping connection");
 615                 icl_pdu_free(request);
 616                 cfiscsi_session_terminate(cs);
 617                 return;
 618         }
 619         ctl_zero_io(io);
 620         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 621         io->io_hdr.io_type = CTL_IO_TASK;
 622         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 623         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
 624         io->io_hdr.nexus.targ_target.id = 0;
 625         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhstmr->bhstmr_lun);
 626         io->io_hdr.nexus.lun_map_fn = cfiscsi_map_lun;
 627         io->io_hdr.nexus.lun_map_arg = cs;
 628         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
 629
 630         switch (bhstmr->bhstmr_function & ~0x80) {
 631         case BHSTMR_FUNCTION_ABORT_TASK:
 632 #if 0
 633                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
 634 #endif
 635                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
 636                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
 637                 break;
 638         case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
 639 #if 0
 640                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
 641 #endif
 642                 io->taskio.task_action = CTL_TASK_LUN_RESET;
 643                 break;
 644         case BHSTMR_FUNCTION_TARGET_WARM_RESET:
 645 #if 0
 646                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
 647 #endif
 648                 io->taskio.task_action = CTL_TASK_TARGET_RESET;
 649                 break;
 650         default:
 651                 CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
 652                     bhstmr->bhstmr_function & ~0x80);
 653                 ctl_free_io(io);
 654
 655                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 656                 if (response == NULL) {
 657                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 658                             "dropping connection");
 659                         icl_pdu_free(request);
 660                         cfiscsi_session_terminate(cs);
 661                         return;
 662                 }
 663                 bhstmr2 = (struct iscsi_bhs_task_management_response *)
 664                     response->ip_bhs;
 665                 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
 666                 bhstmr2->bhstmr_flags = 0x80;
 667                 bhstmr2->bhstmr_response =
 668                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
 669                 bhstmr2->bhstmr_initiator_task_tag =
 670                     bhstmr->bhstmr_initiator_task_tag;
 671                 icl_pdu_free(request);
 672                 cfiscsi_pdu_queue(response);
 673                 return;
 674         }
 675
 676         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 677         error = ctl_queue(io);
 678         if (error != CTL_RETVAL_COMPLETE) {
 679                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 680                     "dropping connection", error);
 681                 ctl_free_io(io);
 682                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 683                 icl_pdu_free(request);
 684                 cfiscsi_session_terminate(cs);
 685         }
 686 }
 687
 688 static bool
 689 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
 690 {
 691         struct iscsi_bhs_data_out *bhsdo;
 692         struct cfiscsi_session *cs;
 693         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
 694         size_t copy_len, len, off, buffer_offset;
 695         int ctl_sg_count;
 696         union ctl_io *io;
 697
 698         cs = PDU_SESSION(request);
 699
 700         KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 701             ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
 702             (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 703             ISCSI_BHS_OPCODE_SCSI_COMMAND,
 704             ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
 705
 706         /*
 707          * We're only using fields common for Data-Out and SCSI Command PDUs.
 708          */
 709         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 710
 711         io = cdw->cdw_ctl_io;
 712         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 713             ("CTL_FLAG_DATA_IN"));
 714
 715 #if 0
 716         CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
 717             request->ip_data_len, io->scsiio.kern_total_len);
 718 #endif
 719
 720         if (io->scsiio.kern_sg_entries > 0) {
 721                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
 722                 ctl_sg_count = io->scsiio.kern_sg_entries;
 723         } else {
 724                 ctl_sglist = &ctl_sg_entry;
 725                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
 726                 ctl_sglist->len = io->scsiio.kern_data_len;
 727                 ctl_sg_count = 1;
 728         }
 729
 730         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 731             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 732                 buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
 733         else
 734                 buffer_offset = 0;
 735         len = icl_pdu_data_segment_length(request);
 736
 737         /*
 738          * Make sure the offset, as sent by the initiator, matches the offset
 739          * we're supposed to be at in the scatter-gather list.
 740          */
 741         if (buffer_offset >
 742             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
 743             buffer_offset + len <=
 744             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
 745                 CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
 746                     "expected %zd; dropping connection", buffer_offset,
 747                     (size_t)io->scsiio.kern_rel_offset +
 748                     (size_t)io->scsiio.ext_data_filled);
 749                 ctl_set_data_phase_error(&io->scsiio);
 750                 cfiscsi_session_terminate(cs);
 751                 return (true);
 752         }
 753
 754         /*
 755          * This is the offset within the PDU data segment, as opposed
 756          * to buffer_offset, which is the offset within the task (SCSI
 757          * command).
 758          */
 759         off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
 760             buffer_offset;
 761
 762         /*
 763          * Iterate over the scatter/gather segments, filling them with data
 764          * from the PDU data segment.  Note that this can get called multiple
 765          * times for one SCSI command; the cdw structure holds state for the
 766          * scatter/gather list.
 767          */
 768         for (;;) {
 769                 KASSERT(cdw->cdw_sg_index < ctl_sg_count,
 770                     ("cdw->cdw_sg_index >= ctl_sg_count"));
 771                 if (cdw->cdw_sg_len == 0) {
 772                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
 773                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
 774                 }
 775                 KASSERT(off <= len, ("len > off"));
 776                 copy_len = len - off;
 777                 if (copy_len > cdw->cdw_sg_len)
 778                         copy_len = cdw->cdw_sg_len;
 779
 780                 icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
 781                 cdw->cdw_sg_addr += copy_len;
 782                 cdw->cdw_sg_len -= copy_len;
 783                 off += copy_len;
 784                 io->scsiio.ext_data_filled += copy_len;
 785
 786                 if (cdw->cdw_sg_len == 0) {
 787                         /*
 788                          * End of current segment.
 789                          */
 790                         if (cdw->cdw_sg_index == ctl_sg_count - 1) {
 791                                 /*
 792                                  * Last segment in scatter/gather list.
 793                                  */
 794                                 break;
 795                         }
 796                         cdw->cdw_sg_index++;
 797                 }
 798
 799                 if (off == len) {
 800                         /*
 801                          * End of PDU payload.
 802                          */
 803                         break;
 804                 }
 805         }
 806
 807         if (len > off) {
 808                 /*
 809                  * In case of unsolicited data, it's possible that the buffer
 810                  * provided by CTL is smaller than negotiated FirstBurstLength.
 811                  * Just ignore the superfluous data; will ask for them with R2T
 812                  * on next call to cfiscsi_datamove().
 813                  *
 814                  * This obviously can only happen with SCSI Command PDU.
 815                  */
 816                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 817                     ISCSI_BHS_OPCODE_SCSI_COMMAND)
 818                         return (true);
 819
 820                 CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
 821                     "expected %zd; dropping connection",
 822                     icl_pdu_data_segment_length(request), off);
 823                 ctl_set_data_phase_error(&io->scsiio);
 824                 cfiscsi_session_terminate(cs);
 825                 return (true);
 826         }
 827
 828         if (io->scsiio.ext_data_filled == io->scsiio.kern_data_len &&
 829             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
 830                 CFISCSI_SESSION_WARN(cs, "got the final packet without "
 831                     "the F flag; flags = 0x%x; dropping connection",
 832                     bhsdo->bhsdo_flags);
 833                 ctl_set_data_phase_error(&io->scsiio);
 834                 cfiscsi_session_terminate(cs);
 835                 return (true);
 836         }
 837
 838         if (io->scsiio.ext_data_filled != io->scsiio.kern_data_len &&
 839             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
 840                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 841                     ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 842                         CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
 843                             "transmitted size was %zd bytes instead of %d; "
 844                             "dropping connection",
 845                             (size_t)io->scsiio.ext_data_filled,
 846                             io->scsiio.kern_data_len);
 847                         ctl_set_data_phase_error(&io->scsiio);
 848                         cfiscsi_session_terminate(cs);
 849                         return (true);
 850                 } else {
 851                         /*
 852                          * For SCSI Command PDU, this just means we need to
 853                          * solicit more data by sending R2T.
 854                          */
 855                         return (false);
 856                 }
 857         }
 858
 859         if (io->scsiio.ext_data_filled == io->scsiio.kern_data_len) {
 860 #if 0
 861                 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
 862                     "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
 863 #endif
 864
 865                 return (true);
 866         }
 867
 868         return (false);
 869 }
 870
 871 static void
 872 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
 873 {
 874         struct iscsi_bhs_data_out *bhsdo;
 875         struct cfiscsi_session *cs;
 876         struct cfiscsi_data_wait *cdw = NULL;
 877         union ctl_io *io;
 878         bool done;
 879
 880         cs = PDU_SESSION(request);
 881         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 882
 883         CFISCSI_SESSION_LOCK(cs);
 884         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
 885 #if 0
 886                 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
 887                     "ttt 0x%x, itt 0x%x",
 888                     bhsdo->bhsdo_target_transfer_tag,
 889                     bhsdo->bhsdo_initiator_task_tag,
 890                     cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
 891 #endif
 892                 if (bhsdo->bhsdo_target_transfer_tag ==
 893                     cdw->cdw_target_transfer_tag)
 894                         break;
 895         }
 896         CFISCSI_SESSION_UNLOCK(cs);
 897         if (cdw == NULL) {
 898                 CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
 899                     "0x%x, not found; dropping connection",
 900                     bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
 901                 icl_pdu_free(request);
 902                 cfiscsi_session_terminate(cs);
 903                 return;
 904         }
 905
 906         io = cdw->cdw_ctl_io;
 907         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 908             ("CTL_FLAG_DATA_IN"));
 909
 910         done = cfiscsi_handle_data_segment(request, cdw);
 911         if (done) {
 912                 CFISCSI_SESSION_LOCK(cs);
 913                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
 914                 CFISCSI_SESSION_UNLOCK(cs);
 915                 uma_zfree(cfiscsi_data_wait_zone, cdw);
 916                 io->scsiio.be_move_done(io);
 917         }
 918
 919         icl_pdu_free(request);
 920 }
 921
 922 static void
 923 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
 924 {
 925         struct iscsi_bhs_logout_request *bhslr;
 926         struct iscsi_bhs_logout_response *bhslr2;
 927         struct icl_pdu *response;
 928         struct cfiscsi_session *cs;
 929
 930         cs = PDU_SESSION(request);
 931         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 932         switch (bhslr->bhslr_reason & 0x7f) {
 933         case BHSLR_REASON_CLOSE_SESSION:
 934         case BHSLR_REASON_CLOSE_CONNECTION:
 935                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 936                 if (response == NULL) {
 937                         CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
 938                         icl_pdu_free(request);
 939                         cfiscsi_session_terminate(cs);
 940                         return;
 941                 }
 942                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 943                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 944                 bhslr2->bhslr_flags = 0x80;
 945                 bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
 946                 bhslr2->bhslr_initiator_task_tag =
 947                     bhslr->bhslr_initiator_task_tag;
 948                 icl_pdu_free(request);
 949                 cfiscsi_pdu_queue(response);
 950                 cfiscsi_session_terminate(cs);
 951                 break;
 952         case BHSLR_REASON_REMOVE_FOR_RECOVERY:
 953                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 954                 if (response == NULL) {
 955                         CFISCSI_SESSION_WARN(cs,
 956                             "failed to allocate memory; dropping connection");
 957                         icl_pdu_free(request);
 958                         cfiscsi_session_terminate(cs);
 959                         return;
 960                 }
 961                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 962                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 963                 bhslr2->bhslr_flags = 0x80;
 964                 bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
 965                 bhslr2->bhslr_initiator_task_tag =
 966                     bhslr->bhslr_initiator_task_tag;
 967                 icl_pdu_free(request);
 968                 cfiscsi_pdu_queue(response);
 969                 break;
 970         default:
 971                 CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
 972                     bhslr->bhslr_reason);
 973                 icl_pdu_free(request);
 974                 cfiscsi_session_terminate(cs);
 975                 break;
 976         }
 977 }
 978
 979 static void
 980 cfiscsi_callout(void *context)
 981 {
 982         struct icl_pdu *cp;
 983         struct iscsi_bhs_nop_in *bhsni;
 984         struct cfiscsi_session *cs;
 985
 986         cs = context;
 987
 988         if (cs->cs_terminating)
 989                 return;
 990
 991         callout_schedule(&cs->cs_callout, 1 * hz);
 992
 993         atomic_add_int(&cs->cs_timeout, 1);
 994
 995 #ifdef ICL_KERNEL_PROXY
 996         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 997                 if (cs->cs_timeout > login_timeout) {
 998                         CFISCSI_SESSION_WARN(cs, "login timed out after "
 999                             "%d seconds; dropping connection", cs->cs_timeout);
1000                         cfiscsi_session_terminate(cs);
1001                 }
1002                 return;
1003         }
1004 #endif
1005
1006         if (cs->cs_timeout >= ping_timeout) {
1007                 CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1008                     "dropping connection",  ping_timeout);
1009                 cfiscsi_session_terminate(cs);
1010                 return;
1011         }
1012
1013         /*
1014          * If the ping was reset less than one second ago - which means
1015          * that we've received some PDU during the last second - assume
1016          * the traffic flows correctly and don't bother sending a NOP-Out.
1017          *
1018          * (It's 2 - one for one second, and one for incrementing is_timeout
1019          * earlier in this routine.)
1020          */
1021         if (cs->cs_timeout < 2)
1022                 return;
1023
1024         cp = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
1025         if (cp == NULL) {
1026                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1027                 return;
1028         }
1029         bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1030         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1031         bhsni->bhsni_flags = 0x80;
1032         bhsni->bhsni_initiator_task_tag = 0xffffffff;
1033
1034         cfiscsi_pdu_queue(cp);
1035 }
1036
1037 static void
1038 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1039 {
1040         struct cfiscsi_data_wait *cdw, *tmpcdw;
1041         union ctl_io *io;
1042         int error, last;
1043
1044 #ifdef notyet
1045         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1046         if (io == NULL) {
1047                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io");
1048                 return;
1049         }
1050         ctl_zero_io(io);
1051         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = NULL;
1052         io->io_hdr.io_type = CTL_IO_TASK;
1053         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1054         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1055         io->io_hdr.nexus.targ_target.id = 0;
1056         io->io_hdr.nexus.targ_lun = lun;
1057         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1058         io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
1059         error = ctl_queue(io);
1060         if (error != CTL_RETVAL_COMPLETE) {
1061                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1062                 ctl_free_io(io);
1063         }
1064 #else
1065         /*
1066          * CTL doesn't currently support CTL_TASK_ABORT_TASK_SET, so instead
1067          * just iterate over tasks that are waiting for something - data - and
1068          * terminate those.
1069          */
1070         CFISCSI_SESSION_LOCK(cs);
1071         TAILQ_FOREACH_SAFE(cdw,
1072             &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
1073                 io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1074                 if (io == NULL) {
1075                         CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io");
1076                         return;
1077                 }
1078                 ctl_zero_io(io);
1079                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = NULL;
1080                 io->io_hdr.io_type = CTL_IO_TASK;
1081                 io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1082                 io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1083                 io->io_hdr.nexus.targ_target.id = 0;
1084                 //io->io_hdr.nexus.targ_lun = lun; /* Not needed? */
1085                 io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1086                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
1087                 io->taskio.tag_num = cdw->cdw_initiator_task_tag;
1088                 error = ctl_queue(io);
1089                 if (error != CTL_RETVAL_COMPLETE) {
1090                         CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1091                         ctl_free_io(io);
1092                         return;
1093                 }
1094 #if 0
1095                 CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task tag "
1096                     "0x%x", cdw->cdw_initiator_task_tag);
1097 #endif
1098                 /*
1099                  * Set nonzero port status; this prevents backends from
1100                  * assuming that the data transfer actually succeeded
1101                  * and writing uninitialized data to disk.
1102                  */
1103                 cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
1104                 cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1105                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1106                 uma_zfree(cfiscsi_data_wait_zone, cdw);
1107         }
1108         CFISCSI_SESSION_UNLOCK(cs);
1109 #endif
1110
1111         /*
1112          * Wait for CTL to terminate all the tasks.
1113          */
1114         for (;;) {
1115                 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1116                 last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1117                 if (last != 0)
1118                         break;
1119                 CFISCSI_SESSION_WARN(cs, "waiting for CTL to terminate tasks, "
1120                     "%d remaining", cs->cs_outstanding_ctl_pdus);
1121                 pause("cfiscsi_terminate", 1);
1122         }
1123 }
1124
1125 static void
1126 cfiscsi_maintenance_thread(void *arg)
1127 {
1128         struct cfiscsi_session *cs;
1129
1130         cs = arg;
1131
1132         for (;;) {
1133                 CFISCSI_SESSION_LOCK(cs);
1134                 if (cs->cs_terminating == false)
1135                         cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1136                 CFISCSI_SESSION_UNLOCK(cs);
1137
1138                 if (cs->cs_terminating) {
1139
1140                         /*
1141                          * We used to wait up to 30 seconds to deliver queued
1142                          * PDUs to the initiator.  We also tried hard to deliver
1143                          * SCSI Responses for the aborted PDUs.  We don't do
1144                          * that anymore.  We might need to revisit that.
1145                          */
1146                         callout_drain(&cs->cs_callout);
1147                         icl_conn_shutdown(cs->cs_conn);
1148                         icl_conn_close(cs->cs_conn);
1149
1150                         /*
1151                          * At this point ICL receive thread is no longer
1152                          * running; no new tasks can be queued.
1153                          */
1154                         cfiscsi_session_terminate_tasks(cs);
1155                         cfiscsi_session_delete(cs);
1156                         kthread_exit();
1157                         return;
1158                 }
1159                 CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1160         }
1161 }
1162
1163 static void
1164 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1165 {
1166
1167         if (cs->cs_terminating)
1168                 return;
1169         cs->cs_terminating = true;
1170         cv_signal(&cs->cs_maintenance_cv);
1171 #ifdef ICL_KERNEL_PROXY
1172         cv_signal(&cs->cs_login_cv);
1173 #endif
1174 }
1175
1176 static int
1177 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1178 {
1179         int error, i;
1180         struct cfiscsi_softc *softc;
1181
1182         KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1183
1184         softc = &cfiscsi_softc;
1185
1186         mtx_lock(&softc->lock);
1187         for (i = 0; i < softc->max_initiators; i++) {
1188                 if (softc->ctl_initids[i] == 0)
1189                         break;
1190         }
1191         if (i == softc->max_initiators) {
1192                 CFISCSI_SESSION_WARN(cs, "too many concurrent sessions (%d)",
1193                     softc->max_initiators);
1194                 mtx_unlock(&softc->lock);
1195                 return (1);
1196         }
1197         softc->ctl_initids[i] = 1;
1198         mtx_unlock(&softc->lock);
1199
1200 #if 0
1201         CFISCSI_SESSION_DEBUG(cs, "adding initiator id %d, max %d",
1202             i, softc->max_initiators);
1203 #endif
1204         cs->cs_ctl_initid = i;
1205         error = ctl_add_initiator(0x0, cs->cs_target->ct_port.targ_port, cs->cs_ctl_initid);
1206         if (error != 0) {
1207                 CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d", error);
1208                 mtx_lock(&softc->lock);
1209                 softc->ctl_initids[cs->cs_ctl_initid] = 0;
1210                 mtx_unlock(&softc->lock);
1211                 cs->cs_ctl_initid = -1;
1212                 return (1);
1213         }
1214
1215         return (0);
1216 }
1217
1218 static void
1219 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1220 {
1221         int error;
1222         struct cfiscsi_softc *softc;
1223
1224         if (cs->cs_ctl_initid == -1)
1225                 return;
1226
1227         softc = &cfiscsi_softc;
1228
1229         error = ctl_remove_initiator(cs->cs_target->ct_port.targ_port, cs->cs_ctl_initid);
1230         if (error != 0) {
1231                 CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1232                     error);
1233         }
1234         mtx_lock(&softc->lock);
1235         softc->ctl_initids[cs->cs_ctl_initid] = 0;
1236         mtx_unlock(&softc->lock);
1237         cs->cs_ctl_initid = -1;
1238 }
1239
1240 static struct cfiscsi_session *
1241 cfiscsi_session_new(struct cfiscsi_softc *softc)
1242 {
1243         struct cfiscsi_session *cs;
1244         int error;
1245
1246         cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1247         if (cs == NULL) {
1248                 CFISCSI_WARN("malloc failed");
1249                 return (NULL);
1250         }
1251         cs->cs_ctl_initid = -1;
1252
1253         refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1254         TAILQ_INIT(&cs->cs_waiting_for_data_out);
1255         mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1256         cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1257 #ifdef ICL_KERNEL_PROXY
1258         cv_init(&cs->cs_login_cv, "cfiscsi_login");
1259 #endif
1260
1261         cs->cs_conn = icl_conn_new("cfiscsi", &cs->cs_lock);
1262         cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1263         cs->cs_conn->ic_error = cfiscsi_error_callback;
1264         cs->cs_conn->ic_prv0 = cs;
1265
1266         error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1267         if (error != 0) {
1268                 CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1269                 free(cs, M_CFISCSI);
1270                 return (NULL);
1271         }
1272
1273         mtx_lock(&softc->lock);
1274         cs->cs_id = softc->last_session_id + 1;
1275         softc->last_session_id++;
1276         mtx_unlock(&softc->lock);
1277
1278         mtx_lock(&softc->lock);
1279         TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1280         mtx_unlock(&softc->lock);
1281
1282         /*
1283          * Start pinging the initiator.
1284          */
1285         callout_init(&cs->cs_callout, 1);
1286         callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1287
1288         return (cs);
1289 }
1290
1291 static void
1292 cfiscsi_session_delete(struct cfiscsi_session *cs)
1293 {
1294         struct cfiscsi_softc *softc;
1295
1296         softc = &cfiscsi_softc;
1297
1298         KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1299             ("destroying session with outstanding CTL pdus"));
1300         KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1301             ("destroying session with non-empty queue"));
1302
1303         cfiscsi_session_unregister_initiator(cs);
1304         if (cs->cs_target != NULL)
1305                 cfiscsi_target_release(cs->cs_target);
1306         icl_conn_close(cs->cs_conn);
1307         icl_conn_free(cs->cs_conn);
1308
1309         mtx_lock(&softc->lock);
1310         TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1311         mtx_unlock(&softc->lock);
1312
1313         free(cs, M_CFISCSI);
1314 }
1315
1316 int
1317 cfiscsi_init(void)
1318 {
1319         struct cfiscsi_softc *softc;
1320         int retval;
1321
1322         softc = &cfiscsi_softc;
1323         retval = 0;
1324         bzero(softc, sizeof(*softc));
1325         mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1326
1327 #ifdef ICL_KERNEL_PROXY
1328         cv_init(&softc->accept_cv, "cfiscsi_accept");
1329 #endif
1330         TAILQ_INIT(&softc->sessions);
1331         TAILQ_INIT(&softc->targets);
1332
1333         softc->max_initiators = CTL_MAX_INIT_PER_PORT;
1334
1335         cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1336             sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1337             UMA_ALIGN_PTR, 0);
1338
1339         return (0);
1340 }
1341
1342 #ifdef ICL_KERNEL_PROXY
1343 static void
1344 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1345 {
1346         struct cfiscsi_session *cs;
1347
1348         cs = cfiscsi_session_new(&cfiscsi_softc);
1349         if (cs == NULL) {
1350                 CFISCSI_WARN("failed to create session");
1351                 return;
1352         }
1353
1354         icl_conn_handoff_sock(cs->cs_conn, so);
1355         cs->cs_initiator_sa = sa;
1356         cs->cs_portal_id = portal_id;
1357         cs->cs_waiting_for_ctld = true;
1358         cv_signal(&cfiscsi_softc.accept_cv);
1359 }
1360 #endif
1361
1362 static void
1363 cfiscsi_online(void *arg)
1364 {
1365         struct cfiscsi_softc *softc;
1366         struct cfiscsi_target *ct;
1367         int online;
1368
1369         ct = (struct cfiscsi_target *)arg;
1370         softc = ct->ct_softc;
1371
1372         mtx_lock(&softc->lock);
1373         if (ct->ct_online) {
1374                 mtx_unlock(&softc->lock);
1375                 return;
1376         }
1377         ct->ct_online = 1;
1378         online = softc->online++;
1379         mtx_unlock(&softc->lock);
1380         if (online > 0)
1381                 return;
1382
1383 #ifdef ICL_KERNEL_PROXY
1384         if (softc->listener != NULL)
1385                 icl_listen_free(softc->listener);
1386         softc->listener = icl_listen_new(cfiscsi_accept);
1387 #endif
1388 }
1389
1390 static void
1391 cfiscsi_offline(void *arg)
1392 {
1393         struct cfiscsi_softc *softc;
1394         struct cfiscsi_target *ct;
1395         struct cfiscsi_session *cs;
1396         int online;
1397
1398         ct = (struct cfiscsi_target *)arg;
1399         softc = ct->ct_softc;
1400
1401         mtx_lock(&softc->lock);
1402         if (!ct->ct_online) {
1403                 mtx_unlock(&softc->lock);
1404                 return;
1405         }
1406         ct->ct_online = 0;
1407         online = --softc->online;
1408
1409         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1410                 if (cs->cs_target == ct)
1411                         cfiscsi_session_terminate(cs);
1412         }
1413         mtx_unlock(&softc->lock);
1414         if (online > 0)
1415                 return;
1416
1417 #ifdef ICL_KERNEL_PROXY
1418         icl_listen_free(softc->listener);
1419         softc->listener = NULL;
1420 #endif
1421 }
1422
1423 static void
1424 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1425 {
1426         struct cfiscsi_softc *softc;
1427         struct cfiscsi_session *cs;
1428         struct cfiscsi_target *ct;
1429         struct ctl_iscsi_handoff_params *cihp;
1430         int error;
1431
1432         cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1433         softc = &cfiscsi_softc;
1434
1435         CFISCSI_DEBUG("new connection from %s (%s) to %s",
1436             cihp->initiator_name, cihp->initiator_addr,
1437             cihp->target_name);
1438
1439         ct = cfiscsi_target_find(softc, cihp->target_name);
1440         if (ct == NULL) {
1441                 ci->status = CTL_ISCSI_ERROR;
1442                 snprintf(ci->error_str, sizeof(ci->error_str),
1443                     "%s: target not found", __func__);
1444                 return;
1445         }
1446
1447         if (ct->ct_online == 0) {
1448                 ci->status = CTL_ISCSI_ERROR;
1449                 snprintf(ci->error_str, sizeof(ci->error_str),
1450                     "%s: port offline", __func__);
1451                 cfiscsi_target_release(ct);
1452                 return;
1453         }
1454
1455 #ifdef ICL_KERNEL_PROXY
1456         if (cihp->socket > 0 && cihp->connection_id > 0) {
1457                 snprintf(ci->error_str, sizeof(ci->error_str),
1458                     "both socket and connection_id set");
1459                 ci->status = CTL_ISCSI_ERROR;
1460                 cfiscsi_target_release(ct);
1461                 return;
1462         }
1463         if (cihp->socket == 0) {
1464                 mtx_lock(&cfiscsi_softc.lock);
1465                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1466                         if (cs->cs_id == cihp->socket)
1467                                 break;
1468                 }
1469                 if (cs == NULL) {
1470                         mtx_unlock(&cfiscsi_softc.lock);
1471                         snprintf(ci->error_str, sizeof(ci->error_str),
1472                             "connection not found");
1473                         ci->status = CTL_ISCSI_ERROR;
1474                         cfiscsi_target_release(ct);
1475                         return;
1476                 }
1477                 mtx_unlock(&cfiscsi_softc.lock);
1478         } else {
1479 #endif
1480                 cs = cfiscsi_session_new(softc);
1481                 if (cs == NULL) {
1482                         ci->status = CTL_ISCSI_ERROR;
1483                         snprintf(ci->error_str, sizeof(ci->error_str),
1484                             "%s: cfiscsi_session_new failed", __func__);
1485                         cfiscsi_target_release(ct);
1486                         return;
1487                 }
1488 #ifdef ICL_KERNEL_PROXY
1489         }
1490 #endif
1491         cs->cs_target = ct;
1492
1493         /*
1494          * First PDU of Full Feature phase has the same CmdSN as the last
1495          * PDU from the Login Phase received from the initiator.  Thus,
1496          * the -1 below.
1497          */
1498         cs->cs_portal_group_tag = cihp->portal_group_tag;
1499         cs->cs_cmdsn = cihp->cmdsn;
1500         cs->cs_statsn = cihp->statsn;
1501         cs->cs_max_data_segment_length = cihp->max_recv_data_segment_length;
1502         cs->cs_max_burst_length = cihp->max_burst_length;
1503         cs->cs_immediate_data = !!cihp->immediate_data;
1504         if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1505                 cs->cs_conn->ic_header_crc32c = true;
1506         if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1507                 cs->cs_conn->ic_data_crc32c = true;
1508
1509         strlcpy(cs->cs_initiator_name,
1510             cihp->initiator_name, sizeof(cs->cs_initiator_name));
1511         strlcpy(cs->cs_initiator_addr,
1512             cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1513         strlcpy(cs->cs_initiator_alias,
1514             cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1515
1516 #ifdef ICL_KERNEL_PROXY
1517         if (cihp->socket > 0) {
1518 #endif
1519                 error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1520                 if (error != 0) {
1521                         cfiscsi_session_delete(cs);
1522                         ci->status = CTL_ISCSI_ERROR;
1523                         snprintf(ci->error_str, sizeof(ci->error_str),
1524                             "%s: icl_conn_handoff failed with error %d",
1525                             __func__, error);
1526                         return;
1527                 }
1528 #ifdef ICL_KERNEL_PROXY
1529         }
1530 #endif
1531
1532         /*
1533          * Register initiator with CTL.
1534          */
1535         cfiscsi_session_register_initiator(cs);
1536
1537 #ifdef ICL_KERNEL_PROXY
1538         cs->cs_login_phase = false;
1539
1540         /*
1541          * First PDU of the Full Feature phase has likely already arrived.
1542          * We have to pick it up and execute properly.
1543          */
1544         if (cs->cs_login_pdu != NULL) {
1545                 CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1546                 cfiscsi_pdu_handle(cs->cs_login_pdu);
1547                 cs->cs_login_pdu = NULL;
1548         }
1549 #endif
1550
1551         ci->status = CTL_ISCSI_OK;
1552 }
1553
1554 static void
1555 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1556 {
1557         struct ctl_iscsi_list_params *cilp;
1558         struct cfiscsi_session *cs;
1559         struct cfiscsi_softc *softc;
1560         struct sbuf *sb;
1561         int error;
1562
1563         cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1564         softc = &cfiscsi_softc;
1565
1566         sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1567         if (sb == NULL) {
1568                 ci->status = CTL_ISCSI_ERROR;
1569                 snprintf(ci->error_str, sizeof(ci->error_str),
1570                     "Unable to allocate %d bytes for iSCSI session list",
1571                     cilp->alloc_len);
1572                 return;
1573         }
1574
1575         sbuf_printf(sb, "<ctlislist>\n");
1576         mtx_lock(&softc->lock);
1577         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1578 #ifdef ICL_KERNEL_PROXY
1579                 if (cs->cs_target == NULL)
1580                         continue;
1581 #endif
1582                 error = sbuf_printf(sb, "<connection id=\"%d\">"
1583                     "<initiator>%s</initiator>"
1584                     "<initiator_addr>%s</initiator_addr>"
1585                     "<initiator_alias>%s</initiator_alias>"
1586                     "<target>%s</target>"
1587                     "<target_alias>%s</target_alias>"
1588                     "<header_digest>%s</header_digest>"
1589                     "<data_digest>%s</data_digest>"
1590                     "<max_data_segment_length>%zd</max_data_segment_length>"
1591                     "<immediate_data>%d</immediate_data>"
1592                     "<iser>%d</iser>"
1593                     "</connection>\n",
1594                     cs->cs_id,
1595                     cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1596                     cs->cs_target->ct_name, cs->cs_target->ct_alias,
1597                     cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1598                     cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1599                     cs->cs_max_data_segment_length,
1600                     cs->cs_immediate_data,
1601                     cs->cs_conn->ic_iser);
1602                 if (error != 0)
1603                         break;
1604         }
1605         mtx_unlock(&softc->lock);
1606         error = sbuf_printf(sb, "</ctlislist>\n");
1607         if (error != 0) {
1608                 sbuf_delete(sb);
1609                 ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1610                 snprintf(ci->error_str, sizeof(ci->error_str),
1611                     "Out of space, %d bytes is too small", cilp->alloc_len);
1612                 return;
1613         }
1614         sbuf_finish(sb);
1615
1616         error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1617         cilp->fill_len = sbuf_len(sb) + 1;
1618         ci->status = CTL_ISCSI_OK;
1619         sbuf_delete(sb);
1620 }
1621
1622 static void
1623 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1624 {
1625         struct icl_pdu *response;
1626         struct iscsi_bhs_asynchronous_message *bhsam;
1627         struct ctl_iscsi_terminate_params *citp;
1628         struct cfiscsi_session *cs;
1629         struct cfiscsi_softc *softc;
1630         int found = 0;
1631
1632         citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1633         softc = &cfiscsi_softc;
1634
1635         mtx_lock(&softc->lock);
1636         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1637                 if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1638                     strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1639                     strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1640                         continue;
1641
1642                 response = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
1643                 if (response == NULL) {
1644                         /*
1645                          * Oh well.  Just terminate the connection.
1646                          */
1647                 } else {
1648                         bhsam = (struct iscsi_bhs_asynchronous_message *)
1649                             response->ip_bhs;
1650                         bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1651                         bhsam->bhsam_flags = 0x80;
1652                         bhsam->bhsam_0xffffffff = 0xffffffff;
1653                         bhsam->bhsam_async_event =
1654                             BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1655                         cfiscsi_pdu_queue(response);
1656                 }
1657                 cfiscsi_session_terminate(cs);
1658                 found++;
1659         }
1660         mtx_unlock(&softc->lock);
1661
1662         if (found == 0) {
1663                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1664                 snprintf(ci->error_str, sizeof(ci->error_str),
1665                     "No matching connections found");
1666                 return;
1667         }
1668
1669         ci->status = CTL_ISCSI_OK;
1670 }
1671
1672 static void
1673 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1674 {
1675         struct icl_pdu *response;
1676         struct iscsi_bhs_asynchronous_message *bhsam;
1677         struct ctl_iscsi_logout_params *cilp;
1678         struct cfiscsi_session *cs;
1679         struct cfiscsi_softc *softc;
1680         int found = 0;
1681
1682         cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1683         softc = &cfiscsi_softc;
1684
1685         mtx_lock(&softc->lock);
1686         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1687                 if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1688                     strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1689                     strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1690                         continue;
1691
1692                 response = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
1693                 if (response == NULL) {
1694                         ci->status = CTL_ISCSI_ERROR;
1695                         snprintf(ci->error_str, sizeof(ci->error_str),
1696                             "Unable to allocate memory");
1697                         mtx_unlock(&softc->lock);
1698                         return;
1699                 }
1700                 bhsam =
1701                     (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1702                 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1703                 bhsam->bhsam_flags = 0x80;
1704                 bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1705                 bhsam->bhsam_parameter3 = htons(10);
1706                 cfiscsi_pdu_queue(response);
1707                 found++;
1708         }
1709         mtx_unlock(&softc->lock);
1710
1711         if (found == 0) {
1712                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1713                 snprintf(ci->error_str, sizeof(ci->error_str),
1714                     "No matching connections found");
1715                 return;
1716         }
1717
1718         ci->status = CTL_ISCSI_OK;
1719 }
1720
1721 #ifdef ICL_KERNEL_PROXY
1722 static void
1723 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1724 {
1725         struct ctl_iscsi_listen_params *cilp;
1726         struct sockaddr *sa;
1727         int error;
1728
1729         cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1730
1731         if (cfiscsi_softc.listener == NULL) {
1732                 CFISCSI_DEBUG("no listener");
1733                 snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1734                 ci->status = CTL_ISCSI_ERROR;
1735                 return;
1736         }
1737
1738         error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1739         if (error != 0) {
1740                 CFISCSI_DEBUG("getsockaddr, error %d", error);
1741                 snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1742                 ci->status = CTL_ISCSI_ERROR;
1743                 return;
1744         }
1745
1746         error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1747             cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1748         if (error != 0) {
1749                 free(sa, M_SONAME);
1750                 CFISCSI_DEBUG("icl_listen_add, error %d", error);
1751                 snprintf(ci->error_str, sizeof(ci->error_str),
1752                     "icl_listen_add failed, error %d", error);
1753                 ci->status = CTL_ISCSI_ERROR;
1754                 return;
1755         }
1756
1757         ci->status = CTL_ISCSI_OK;
1758 }
1759
1760 static void
1761 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1762 {
1763         struct ctl_iscsi_accept_params *ciap;
1764         struct cfiscsi_session *cs;
1765         int error;
1766
1767         ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1768
1769         mtx_lock(&cfiscsi_softc.lock);
1770         for (;;) {
1771                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1772                         if (cs->cs_waiting_for_ctld)
1773                                 break;
1774                 }
1775                 if (cs != NULL)
1776                         break;
1777                 error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1778                 if (error != 0) {
1779                         mtx_unlock(&cfiscsi_softc.lock);
1780                         snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1781                         ci->status = CTL_ISCSI_ERROR;
1782                         return;
1783                 }
1784         }
1785         mtx_unlock(&cfiscsi_softc.lock);
1786
1787         cs->cs_waiting_for_ctld = false;
1788         cs->cs_login_phase = true;
1789
1790         ciap->connection_id = cs->cs_id;
1791         ciap->portal_id = cs->cs_portal_id;
1792         ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1793         error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1794             cs->cs_initiator_sa->sa_len);
1795         if (error != 0) {
1796                 snprintf(ci->error_str, sizeof(ci->error_str),
1797                     "copyout failed with error %d", error);
1798                 ci->status = CTL_ISCSI_ERROR;
1799                 return;
1800         }
1801
1802         ci->status = CTL_ISCSI_OK;
1803 }
1804
1805 static void
1806 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1807 {
1808         struct ctl_iscsi_send_params *cisp;
1809         struct cfiscsi_session *cs;
1810         struct icl_pdu *ip;
1811         size_t datalen;
1812         void *data;
1813         int error;
1814
1815         cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1816
1817         mtx_lock(&cfiscsi_softc.lock);
1818         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1819                 if (cs->cs_id == cisp->connection_id)
1820                         break;
1821         }
1822         if (cs == NULL) {
1823                 mtx_unlock(&cfiscsi_softc.lock);
1824                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1825                 ci->status = CTL_ISCSI_ERROR;
1826                 return;
1827         }
1828         mtx_unlock(&cfiscsi_softc.lock);
1829
1830 #if 0
1831         if (cs->cs_login_phase == false)
1832                 return (EBUSY);
1833 #endif
1834
1835         if (cs->cs_terminating) {
1836                 snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1837                 ci->status = CTL_ISCSI_ERROR;
1838                 return;
1839         }
1840
1841         datalen = cisp->data_segment_len;
1842         /*
1843          * XXX
1844          */
1845         //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
1846         if (datalen > 65535) {
1847                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1848                 ci->status = CTL_ISCSI_ERROR;
1849                 return;
1850         }
1851         if (datalen > 0) {
1852                 data = malloc(datalen, M_CFISCSI, M_WAITOK);
1853                 error = copyin(cisp->data_segment, data, datalen);
1854                 if (error != 0) {
1855                         free(data, M_CFISCSI);
1856                         snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
1857                         ci->status = CTL_ISCSI_ERROR;
1858                         return;
1859                 }
1860         }
1861
1862         ip = icl_pdu_new_bhs(cs->cs_conn, M_WAITOK);
1863         memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
1864         if (datalen > 0) {
1865                 icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1866                 free(data, M_CFISCSI);
1867         }
1868         CFISCSI_SESSION_LOCK(cs);
1869         icl_pdu_queue(ip);
1870         CFISCSI_SESSION_UNLOCK(cs);
1871         ci->status = CTL_ISCSI_OK;
1872 }
1873
1874 static void
1875 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
1876 {
1877         struct ctl_iscsi_receive_params *cirp;
1878         struct cfiscsi_session *cs;
1879         struct icl_pdu *ip;
1880         void *data;
1881         int error;
1882
1883         cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
1884
1885         mtx_lock(&cfiscsi_softc.lock);
1886         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1887                 if (cs->cs_id == cirp->connection_id)
1888                         break;
1889         }
1890         if (cs == NULL) {
1891                 mtx_unlock(&cfiscsi_softc.lock);
1892                 snprintf(ci->error_str, sizeof(ci->error_str),
1893                     "connection not found");
1894                 ci->status = CTL_ISCSI_ERROR;
1895                 return;
1896         }
1897         mtx_unlock(&cfiscsi_softc.lock);
1898
1899 #if 0
1900         if (is->is_login_phase == false)
1901                 return (EBUSY);
1902 #endif
1903
1904         CFISCSI_SESSION_LOCK(cs);
1905         while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
1906                 error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
1907                 if (error != 0) {
1908                         CFISCSI_SESSION_UNLOCK(cs);
1909                         snprintf(ci->error_str, sizeof(ci->error_str),
1910                             "interrupted by signal");
1911                         ci->status = CTL_ISCSI_ERROR;
1912                         return;
1913                 }
1914         }
1915
1916         if (cs->cs_terminating) {
1917                 CFISCSI_SESSION_UNLOCK(cs);
1918                 snprintf(ci->error_str, sizeof(ci->error_str),
1919                     "connection terminating");
1920                 ci->status = CTL_ISCSI_ERROR;
1921                 return;
1922         }
1923         ip = cs->cs_login_pdu;
1924         cs->cs_login_pdu = NULL;
1925         CFISCSI_SESSION_UNLOCK(cs);
1926
1927         if (ip->ip_data_len > cirp->data_segment_len) {
1928                 icl_pdu_free(ip);
1929                 snprintf(ci->error_str, sizeof(ci->error_str),
1930                     "data segment too big");
1931                 ci->status = CTL_ISCSI_ERROR;
1932                 return;
1933         }
1934
1935         copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
1936         if (ip->ip_data_len > 0) {
1937                 data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
1938                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1939                 copyout(data, cirp->data_segment, ip->ip_data_len);
1940                 free(data, M_CFISCSI);
1941         }
1942
1943         icl_pdu_free(ip);
1944         ci->status = CTL_ISCSI_OK;
1945 }
1946
1947 #endif /* !ICL_KERNEL_PROXY */
1948
1949 static void
1950 cfiscsi_ioctl_port_create(struct ctl_req *req)
1951 {
1952         struct cfiscsi_target *ct;
1953         struct ctl_port *port;
1954         const char *target, *alias, *tag;
1955         ctl_options_t opts;
1956         int retval;
1957
1958         ctl_init_opts(&opts, req->num_args, req->kern_args);
1959         target = ctl_get_opt(&opts, "cfiscsi_target");
1960         alias = ctl_get_opt(&opts, "cfiscsi_target_alias");
1961         tag = ctl_get_opt(&opts, "cfiscsi_portal_group_tag");
1962         if (target == NULL || tag == NULL) {
1963                 ctl_free_opts(&opts);
1964                 req->status = CTL_LUN_ERROR;
1965                 snprintf(req->error_str, sizeof(req->error_str),
1966                     "Missing required argument");
1967                 return;
1968         }
1969         ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias);
1970         if (ct == NULL) {
1971                 ctl_free_opts(&opts);
1972                 req->status = CTL_LUN_ERROR;
1973                 snprintf(req->error_str, sizeof(req->error_str),
1974                     "failed to create target \"%s\"", target);
1975                 return;
1976         }
1977         if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
1978                 cfiscsi_target_release(ct);
1979                 ctl_free_opts(&opts);
1980                 req->status = CTL_LUN_ERROR;
1981                 snprintf(req->error_str, sizeof(req->error_str),
1982                     "target \"%s\" already exist", target);
1983                 return;
1984         }
1985         port = &ct->ct_port;
1986         if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
1987                 goto done;
1988
1989         port->frontend = &cfiscsi_frontend;
1990         port->port_type = CTL_PORT_ISCSI;
1991         /* XXX KDM what should the real number be here? */
1992         port->num_requested_ctl_io = 4096;
1993         port->port_name = "iscsi";
1994         port->virtual_port = strtoul(tag, NULL, 0);
1995         port->port_online = cfiscsi_online;
1996         port->port_offline = cfiscsi_offline;
1997         port->onoff_arg = ct;
1998         port->lun_enable = cfiscsi_lun_enable;
1999         port->lun_disable = cfiscsi_lun_disable;
2000         port->targ_lun_arg = ct;
2001         port->devid = cfiscsi_devid;
2002         port->fe_datamove = cfiscsi_datamove;
2003         port->fe_done = cfiscsi_done;
2004
2005         /* XXX KDM what should we report here? */
2006         /* XXX These should probably be fetched from CTL. */
2007         port->max_targets = 1;
2008         port->max_target_id = 15;
2009
2010         port->options = opts;
2011         STAILQ_INIT(&opts);
2012
2013         retval = ctl_port_register(port, /*master_SC*/ 1);
2014         if (retval != 0) {
2015                 ctl_free_opts(&port->options);
2016                 cfiscsi_target_release(ct);
2017                 req->status = CTL_LUN_ERROR;
2018                 snprintf(req->error_str, sizeof(req->error_str),
2019                     "ctl_frontend_register() failed with error %d", retval);
2020                 return;
2021         }
2022 done:
2023         ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2024         req->status = CTL_LUN_OK;
2025         memcpy(req->kern_args[0].kvalue, &port->targ_port,
2026             sizeof(port->targ_port)); //XXX
2027 }
2028
2029 static void
2030 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2031 {
2032         struct cfiscsi_target *ct;
2033         const char *target;
2034         ctl_options_t opts;
2035
2036         ctl_init_opts(&opts, req->num_args, req->kern_args);
2037         target = ctl_get_opt(&opts, "cfiscsi_target");
2038         if (target == NULL) {
2039                 ctl_free_opts(&opts);
2040                 req->status = CTL_LUN_ERROR;
2041                 snprintf(req->error_str, sizeof(req->error_str),
2042                     "Missing required argument");
2043                 return;
2044         }
2045         ct = cfiscsi_target_find(&cfiscsi_softc, target);
2046         if (ct == NULL) {
2047                 ctl_free_opts(&opts);
2048                 req->status = CTL_LUN_ERROR;
2049                 snprintf(req->error_str, sizeof(req->error_str),
2050                     "can't find target \"%s\"", target);
2051                 return;
2052         }
2053         if (ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE) {
2054                 ctl_free_opts(&opts);
2055                 req->status = CTL_LUN_ERROR;
2056                 snprintf(req->error_str, sizeof(req->error_str),
2057                     "target \"%s\" is already dying", target);
2058                 return;
2059         }
2060         ctl_free_opts(&opts);
2061
2062         ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2063         ctl_port_offline(&ct->ct_port);
2064         cfiscsi_target_release(ct);
2065         cfiscsi_target_release(ct);
2066 }
2067
2068 static int
2069 cfiscsi_ioctl(struct cdev *dev,
2070     u_long cmd, caddr_t addr, int flag, struct thread *td)
2071 {
2072         struct ctl_iscsi *ci;
2073         struct ctl_req *req;
2074
2075         if (cmd == CTL_PORT_REQ) {
2076                 req = (struct ctl_req *)addr;
2077                 switch (req->reqtype) {
2078                 case CTL_REQ_CREATE:
2079                         cfiscsi_ioctl_port_create(req);
2080                         break;
2081                 case CTL_REQ_REMOVE:
2082                         cfiscsi_ioctl_port_remove(req);
2083                         break;
2084                 default:
2085                         req->status = CTL_LUN_ERROR;
2086                         snprintf(req->error_str, sizeof(req->error_str),
2087                             "Unsupported request type %d", req->reqtype);
2088                 }
2089                 return (0);
2090         }
2091
2092         if (cmd != CTL_ISCSI)
2093                 return (ENOTTY);
2094
2095         ci = (struct ctl_iscsi *)addr;
2096         switch (ci->type) {
2097         case CTL_ISCSI_HANDOFF:
2098                 cfiscsi_ioctl_handoff(ci);
2099                 break;
2100         case CTL_ISCSI_LIST:
2101                 cfiscsi_ioctl_list(ci);
2102                 break;
2103         case CTL_ISCSI_TERMINATE:
2104                 cfiscsi_ioctl_terminate(ci);
2105                 break;
2106         case CTL_ISCSI_LOGOUT:
2107                 cfiscsi_ioctl_logout(ci);
2108                 break;
2109 #ifdef ICL_KERNEL_PROXY
2110         case CTL_ISCSI_LISTEN:
2111                 cfiscsi_ioctl_listen(ci);
2112                 break;
2113         case CTL_ISCSI_ACCEPT:
2114                 cfiscsi_ioctl_accept(ci);
2115                 break;
2116         case CTL_ISCSI_SEND:
2117                 cfiscsi_ioctl_send(ci);
2118                 break;
2119         case CTL_ISCSI_RECEIVE:
2120                 cfiscsi_ioctl_receive(ci);
2121                 break;
2122 #else
2123         case CTL_ISCSI_LISTEN:
2124         case CTL_ISCSI_ACCEPT:
2125         case CTL_ISCSI_SEND:
2126         case CTL_ISCSI_RECEIVE:
2127                 ci->status = CTL_ISCSI_ERROR;
2128                 snprintf(ci->error_str, sizeof(ci->error_str),
2129                     "%s: CTL compiled without ICL_KERNEL_PROXY",
2130                     __func__);
2131                 break;
2132 #endif /* !ICL_KERNEL_PROXY */
2133         default:
2134                 ci->status = CTL_ISCSI_ERROR;
2135                 snprintf(ci->error_str, sizeof(ci->error_str),
2136                     "%s: invalid iSCSI request type %d", __func__, ci->type);
2137                 break;
2138         }
2139
2140         return (0);
2141 }
2142
2143 static int
2144 cfiscsi_devid(struct ctl_scsiio *ctsio, int alloc_len)
2145 {
2146         struct cfiscsi_session *cs;
2147         struct scsi_vpd_device_id *devid_ptr;
2148         struct scsi_vpd_id_descriptor *desc, *desc1, *desc2, *desc3, *desc4;
2149         struct scsi_vpd_id_descriptor *desc5;
2150         struct scsi_vpd_id_t10 *t10id;
2151         struct ctl_lun *lun;
2152         const struct icl_pdu *request;
2153         int i, ret;
2154         char *val;
2155         size_t data_len, devid_len, wwnn_len, wwpn_len, lun_name_len;
2156
2157         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
2158         request = ctsio->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2159         cs = PDU_SESSION(request);
2160
2161         wwpn_len = strlen(cs->cs_target->ct_name);
2162         wwpn_len += strlen(",t,0x0001");
2163         wwpn_len += 1; /* '\0' */
2164         if ((wwpn_len % 4) != 0)
2165                 wwpn_len += (4 - (wwpn_len % 4));
2166
2167         wwnn_len = strlen(cs->cs_target->ct_name);
2168         wwnn_len += 1; /* '\0' */
2169         if ((wwnn_len % 4) != 0)
2170                 wwnn_len += (4 - (wwnn_len % 4));
2171
2172         if (lun == NULL) {
2173                 devid_len = CTL_DEVID_MIN_LEN;
2174                 lun_name_len = 0;
2175         } else {
2176                 devid_len = max(CTL_DEVID_MIN_LEN,
2177                     strnlen(lun->be_lun->device_id, CTL_DEVID_LEN));
2178                 lun_name_len = strlen(cs->cs_target->ct_name);
2179                 lun_name_len += strlen(",lun,XXXXXXXX");
2180                 lun_name_len += 1; /* '\0' */
2181                 if ((lun_name_len % 4) != 0)
2182                         lun_name_len += (4 - (lun_name_len % 4));
2183         }
2184
2185         data_len = sizeof(struct scsi_vpd_device_id) +
2186                 sizeof(struct scsi_vpd_id_descriptor) +
2187                 sizeof(struct scsi_vpd_id_t10) + devid_len +
2188                 sizeof(struct scsi_vpd_id_descriptor) + lun_name_len +
2189                 sizeof(struct scsi_vpd_id_descriptor) + wwnn_len +
2190                 sizeof(struct scsi_vpd_id_descriptor) + wwpn_len +
2191                 sizeof(struct scsi_vpd_id_descriptor) +
2192                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
2193                 sizeof(struct scsi_vpd_id_descriptor) +
2194                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
2195
2196         ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
2197         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
2198         ctsio->kern_sg_entries = 0;
2199
2200         if (data_len < alloc_len) {
2201                 ctsio->residual = alloc_len - data_len;
2202                 ctsio->kern_data_len = data_len;
2203                 ctsio->kern_total_len = data_len;
2204         } else {
2205                 ctsio->residual = 0;
2206                 ctsio->kern_data_len = alloc_len;
2207                 ctsio->kern_total_len = alloc_len;
2208         }
2209         ctsio->kern_data_resid = 0;
2210         ctsio->kern_rel_offset = 0;
2211         ctsio->kern_sg_entries = 0;
2212
2213         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
2214         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
2215         desc1 = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
2216             sizeof(struct scsi_vpd_id_t10) + devid_len);
2217         desc2 = (struct scsi_vpd_id_descriptor *)(&desc1->identifier[0] +
2218             lun_name_len);
2219         desc3 = (struct scsi_vpd_id_descriptor *)(&desc2->identifier[0] +
2220             wwnn_len);
2221         desc4 = (struct scsi_vpd_id_descriptor *)(&desc3->identifier[0] +
2222             wwpn_len);
2223         desc5 = (struct scsi_vpd_id_descriptor *)(&desc4->identifier[0] +
2224             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
2225
2226         if (lun != NULL)
2227                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
2228                     lun->be_lun->lun_type;
2229         else
2230                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
2231
2232         devid_ptr->page_code = SVPD_DEVICE_ID;
2233
2234         scsi_ulto2b(data_len - 4, devid_ptr->length);
2235
2236         /*
2237          * We're using a LUN association here.  i.e., this device ID is a
2238          * per-LUN identifier.
2239          */
2240         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_ASCII;
2241         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
2242         desc->length = sizeof(*t10id) + devid_len;
2243         if (lun == NULL || (val = ctl_get_opt(&lun->be_lun->options,
2244             "vendor")) == NULL) {
2245                 strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
2246         } else {
2247                 memset(t10id->vendor, ' ', sizeof(t10id->vendor));
2248                 strncpy(t10id->vendor, val,
2249                     min(sizeof(t10id->vendor), strlen(val)));
2250         }
2251
2252         /*
2253          * If we've actually got a backend, copy the device id from the
2254          * per-LUN data.  Otherwise, set it to all spaces.
2255          */
2256         if (lun != NULL) {
2257                 /*
2258                  * Copy the backend's LUN ID.
2259                  */
2260                 strncpy((char *)t10id->vendor_spec_id,
2261                     (char *)lun->be_lun->device_id, devid_len);
2262         } else {
2263                 /*
2264                  * No backend, set this to spaces.
2265                  */
2266                 memset(t10id->vendor_spec_id, 0x20, devid_len);
2267         }
2268
2269         /*
2270          * desc1 is for the unique LUN name.
2271          *
2272          * XXX: According to SPC-3, LUN must report the same ID through
2273          *      all the ports.  The code below, however, reports the
2274          *      ID only via iSCSI.
2275          */
2276         desc1->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2277         desc1->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN |
2278                 SVPD_ID_TYPE_SCSI_NAME;
2279         desc1->length = lun_name_len;
2280         if (lun != NULL) {
2281                 /*
2282                  * Find the per-target LUN number.
2283                  */
2284                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2285                         if (cs->cs_target->ct_luns[i] == lun->lun)
2286                                 break;
2287                 }
2288                 KASSERT(i < CTL_MAX_LUNS,
2289                     ("lun %jd not found", (uintmax_t)lun->lun));
2290                 ret = snprintf(desc1->identifier, lun_name_len, "%s,lun,%d",
2291                     cs->cs_target->ct_name, i);
2292                 KASSERT(ret > 0 && ret <= lun_name_len, ("bad snprintf"));
2293         } else {
2294                 KASSERT(lun_name_len == 0, ("no lun, but lun_name_len != 0"));
2295         }
2296
2297         /*
2298          * desc2 is for the Target Name.
2299          */
2300         desc2->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2301         desc2->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2302             SVPD_ID_TYPE_SCSI_NAME;
2303         desc2->length = wwnn_len;
2304         snprintf(desc2->identifier, wwnn_len, "%s", cs->cs_target->ct_name);
2305
2306         /*
2307          * desc3 is for the WWPN which is a port asscociation.
2308          */
2309         desc3->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2310         desc3->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2311             SVPD_ID_TYPE_SCSI_NAME;
2312         desc3->length = wwpn_len;
2313         snprintf(desc3->identifier, wwpn_len, "%s,t,0x%4.4x",
2314             cs->cs_target->ct_name, cs->cs_portal_group_tag);
2315
2316         /*
2317          * desc3 is for the Relative Target Port(type 4h) identifier
2318          */
2319         desc4->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_BINARY;
2320         desc4->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2321             SVPD_ID_TYPE_RELTARG;
2322         desc4->length = 4;
2323         desc4->identifier[3] = 1;
2324
2325         /*
2326          * desc4 is for the Target Port Group(type 5h) identifier
2327          */
2328         desc5->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_BINARY;
2329         desc5->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2330             SVPD_ID_TYPE_TPORTGRP;
2331         desc5->length = 4;
2332         desc5->identifier[3] = 1;
2333
2334         ctsio->scsi_status = SCSI_STATUS_OK;
2335
2336         ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
2337         ctsio->be_move_done = ctl_config_move_done;
2338         ctl_datamove((union ctl_io *)ctsio);
2339
2340         return (CTL_RETVAL_COMPLETE);
2341 }
2342
2343 static void
2344 cfiscsi_target_hold(struct cfiscsi_target *ct)
2345 {
2346
2347         refcount_acquire(&ct->ct_refcount);
2348 }
2349
2350 static void
2351 cfiscsi_target_release(struct cfiscsi_target *ct)
2352 {
2353         struct cfiscsi_softc *softc;
2354
2355         softc = ct->ct_softc;
2356         mtx_lock(&softc->lock);
2357         if (refcount_release(&ct->ct_refcount)) {
2358                 TAILQ_REMOVE(&softc->targets, ct, ct_next);
2359                 mtx_unlock(&softc->lock);
2360                 if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2361                         ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2362                         if (ctl_port_deregister(&ct->ct_port) != 0)
2363                                 printf("%s: ctl_port_deregister() failed\n",
2364                                     __func__);
2365                 }
2366                 free(ct, M_CFISCSI);
2367
2368                 return;
2369         }
2370         mtx_unlock(&softc->lock);
2371 }
2372
2373 static struct cfiscsi_target *
2374 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name)
2375 {
2376         struct cfiscsi_target *ct;
2377
2378         mtx_lock(&softc->lock);
2379         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2380                 if (strcmp(name, ct->ct_name) != 0 ||
2381                     ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2382                         continue;
2383                 cfiscsi_target_hold(ct);
2384                 mtx_unlock(&softc->lock);
2385                 return (ct);
2386         }
2387         mtx_unlock(&softc->lock);
2388
2389         return (NULL);
2390 }
2391
2392 static struct cfiscsi_target *
2393 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2394     const char *alias)
2395 {
2396         struct cfiscsi_target *ct, *newct;
2397         int i;
2398
2399         if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2400                 return (NULL);
2401
2402         newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2403
2404         mtx_lock(&softc->lock);
2405         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2406                 if (strcmp(name, ct->ct_name) != 0 ||
2407                     ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2408                         continue;
2409                 cfiscsi_target_hold(ct);
2410                 mtx_unlock(&softc->lock);
2411                 free(newct, M_CFISCSI);
2412                 return (ct);
2413         }
2414
2415         for (i = 0; i < CTL_MAX_LUNS; i++)
2416                 newct->ct_luns[i] = -1;
2417
2418         strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2419         if (alias != NULL)
2420                 strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2421         refcount_init(&newct->ct_refcount, 1);
2422         newct->ct_softc = softc;
2423         TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2424         mtx_unlock(&softc->lock);
2425
2426         return (newct);
2427 }
2428
2429 /*
2430  * Takes LUN from the target space and returns LUN from the CTL space.
2431  */
2432 static uint32_t
2433 cfiscsi_map_lun(void *arg, uint32_t lun)
2434 {
2435         struct cfiscsi_session *cs;
2436
2437         cs = arg;
2438
2439         if (lun >= CTL_MAX_LUNS) {
2440                 CFISCSI_DEBUG("requested lun number %d is higher "
2441                     "than maximum %d", lun, CTL_MAX_LUNS - 1);
2442                 return (0xffffffff);
2443         }
2444
2445         if (cs->cs_target->ct_luns[lun] < 0)
2446                 return (0xffffffff);
2447
2448         return (cs->cs_target->ct_luns[lun]);
2449 }
2450
2451 static int
2452 cfiscsi_target_set_lun(struct cfiscsi_target *ct,
2453     unsigned long lun_id, unsigned long ctl_lun_id)
2454 {
2455
2456         if (lun_id >= CTL_MAX_LUNS) {
2457                 CFISCSI_WARN("requested lun number %ld is higher "
2458                     "than maximum %d", lun_id, CTL_MAX_LUNS - 1);
2459                 return (-1);
2460         }
2461
2462         if (ct->ct_luns[lun_id] >= 0) {
2463                 /*
2464                  * CTL calls cfiscsi_lun_enable() twice for each LUN - once
2465                  * when the LUN is created, and a second time just before
2466                  * the port is brought online; don't emit warnings
2467                  * for that case.
2468                  */
2469                 if (ct->ct_luns[lun_id] == ctl_lun_id)
2470                         return (0);
2471                 CFISCSI_WARN("lun %ld already allocated", lun_id);
2472                 return (-1);
2473         }
2474
2475 #if 0
2476         CFISCSI_DEBUG("adding mapping for lun %ld, target %s "
2477             "to ctl lun %ld", lun_id, ct->ct_name, ctl_lun_id);
2478 #endif
2479
2480         ct->ct_luns[lun_id] = ctl_lun_id;
2481
2482         return (0);
2483 }
2484
2485 static int
2486 cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id)
2487 {
2488         struct cfiscsi_softc *softc;
2489         struct cfiscsi_target *ct;
2490         const char *target = NULL;
2491         const char *lun = NULL;
2492         unsigned long tmp;
2493
2494         ct = (struct cfiscsi_target *)arg;
2495         softc = ct->ct_softc;
2496
2497         target = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options,
2498             "cfiscsi_target");
2499         lun = ctl_get_opt(&control_softc->ctl_luns[lun_id]->be_lun->options,
2500             "cfiscsi_lun");
2501
2502         if (target == NULL && lun == NULL)
2503                 return (0);
2504
2505         if (target == NULL || lun == NULL) {
2506                 CFISCSI_WARN("lun added with cfiscsi_target, but without "
2507                     "cfiscsi_lun, or the other way around; ignoring");
2508                 return (0);
2509         }
2510
2511         if (strcmp(target, ct->ct_name) != 0)
2512                 return (0);
2513
2514         tmp = strtoul(lun, NULL, 10);
2515         cfiscsi_target_set_lun(ct, tmp, lun_id);
2516         return (0);
2517 }
2518
2519 static int
2520 cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
2521 {
2522         struct cfiscsi_softc *softc;
2523         struct cfiscsi_target *ct;
2524         int i;
2525
2526         ct = (struct cfiscsi_target *)arg;
2527         softc = ct->ct_softc;
2528
2529         mtx_lock(&softc->lock);
2530         for (i = 0; i < CTL_MAX_LUNS; i++) {
2531                 if (ct->ct_luns[i] < 0)
2532                         continue;
2533                 if (ct->ct_luns[i] != lun_id)
2534                         continue;
2535                 ct->ct_luns[lun_id] = -1;
2536                 break;
2537         }
2538         mtx_unlock(&softc->lock);
2539         return (0);
2540 }
2541
2542 static void
2543 cfiscsi_datamove_in(union ctl_io *io)
2544 {
2545         struct cfiscsi_session *cs;
2546         struct icl_pdu *request, *response;
2547         const struct iscsi_bhs_scsi_command *bhssc;
2548         struct iscsi_bhs_data_in *bhsdi;
2549         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2550         size_t len, expected_len, sg_len, buffer_offset;
2551         const char *sg_addr;
2552         int ctl_sg_count, error, i;
2553
2554         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2555         cs = PDU_SESSION(request);
2556
2557         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2558         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2559             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2560             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2561
2562         if (io->scsiio.kern_sg_entries > 0) {
2563                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2564                 ctl_sg_count = io->scsiio.kern_sg_entries;
2565         } else {
2566                 ctl_sglist = &ctl_sg_entry;
2567                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2568                 ctl_sglist->len = io->scsiio.kern_data_len;
2569                 ctl_sg_count = 1;
2570         }
2571
2572         /*
2573          * This is the total amount of data to be transferred within the current
2574          * SCSI command.  We need to record it so that we can properly report
2575          * underflow/underflow.
2576          */
2577         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2578
2579         /*
2580          * This is the offset within the current SCSI command; for the first
2581          * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2582          * it will be the sum of lengths of previous ones.
2583          */
2584         buffer_offset = io->scsiio.kern_rel_offset;
2585
2586         /*
2587          * This is the transfer length expected by the initiator.  In theory,
2588          * it could be different from the correct amount of data from the SCSI
2589          * point of view, even if that doesn't make any sense.
2590          */
2591         expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2592 #if 0
2593         if (expected_len != io->scsiio.kern_total_len) {
2594                 CFISCSI_SESSION_DEBUG(cs, "expected transfer length %zd, "
2595                     "actual length %zd", expected_len,
2596                     (size_t)io->scsiio.kern_total_len);
2597         }
2598 #endif
2599
2600         if (buffer_offset >= expected_len) {
2601 #if 0
2602                 CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2603                     "already sent the expected len", buffer_offset);
2604 #endif
2605                 io->scsiio.be_move_done(io);
2606                 return;
2607         }
2608
2609         i = 0;
2610         sg_addr = NULL;
2611         sg_len = 0;
2612         response = NULL;
2613         bhsdi = NULL;
2614         for (;;) {
2615                 if (response == NULL) {
2616                         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2617                         if (response == NULL) {
2618                                 CFISCSI_SESSION_WARN(cs, "failed to "
2619                                     "allocate memory; dropping connection");
2620                                 ctl_set_busy(&io->scsiio);
2621                                 io->scsiio.be_move_done(io);
2622                                 cfiscsi_session_terminate(cs);
2623                                 return;
2624                         }
2625                         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2626                         bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2627                         bhsdi->bhsdi_initiator_task_tag =
2628                             bhssc->bhssc_initiator_task_tag;
2629                         bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2630                         PDU_EXPDATASN(request)++;
2631                         bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2632                 }
2633
2634                 KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2635                 if (sg_len == 0) {
2636                         sg_addr = ctl_sglist[i].addr;
2637                         sg_len = ctl_sglist[i].len;
2638                         KASSERT(sg_len > 0, ("sg_len <= 0"));
2639                 }
2640
2641                 len = sg_len;
2642
2643                 /*
2644                  * Truncate to maximum data segment length.
2645                  */
2646                 KASSERT(response->ip_data_len < cs->cs_max_data_segment_length,
2647                     ("ip_data_len %zd >= max_data_segment_length %zd",
2648                     response->ip_data_len, cs->cs_max_data_segment_length));
2649                 if (response->ip_data_len + len >
2650                     cs->cs_max_data_segment_length) {
2651                         len = cs->cs_max_data_segment_length -
2652                             response->ip_data_len;
2653                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2654                             len, sg_len));
2655                 }
2656
2657                 /*
2658                  * Truncate to expected data transfer length.
2659                  */
2660                 KASSERT(buffer_offset + response->ip_data_len < expected_len,
2661                     ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2662                     buffer_offset, response->ip_data_len, expected_len));
2663                 if (buffer_offset + response->ip_data_len + len > expected_len) {
2664                         CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2665                             "to expected data transfer length %zd",
2666                             buffer_offset + response->ip_data_len + len, expected_len);
2667                         len = expected_len - (buffer_offset + response->ip_data_len);
2668                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2669                             len, sg_len));
2670                 }
2671
2672                 error = icl_pdu_append_data(response, sg_addr, len, M_NOWAIT);
2673                 if (error != 0) {
2674                         CFISCSI_SESSION_WARN(cs, "failed to "
2675                             "allocate memory; dropping connection");
2676                         icl_pdu_free(response);
2677                         ctl_set_busy(&io->scsiio);
2678                         io->scsiio.be_move_done(io);
2679                         cfiscsi_session_terminate(cs);
2680                         return;
2681                 }
2682                 sg_addr += len;
2683                 sg_len -= len;
2684
2685                 KASSERT(buffer_offset + request->ip_data_len <= expected_len,
2686                     ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2687                     buffer_offset, request->ip_data_len, expected_len));
2688                 if (buffer_offset + request->ip_data_len == expected_len) {
2689                         /*
2690                          * Already have the amount of data the initiator wanted.
2691                          */
2692                         break;
2693                 }
2694
2695                 if (sg_len == 0) {
2696                         /*
2697                          * End of scatter-gather segment;
2698                          * proceed to the next one...
2699                          */
2700                         if (i == ctl_sg_count - 1) {
2701                                 /*
2702                                  * ... unless this was the last one.
2703                                  */
2704                                 break;
2705                         }
2706                         i++;
2707                 }
2708
2709                 if (response->ip_data_len == cs->cs_max_data_segment_length) {
2710                         /*
2711                          * Can't stuff more data into the current PDU;
2712                          * queue it.  Note that's not enough to check
2713                          * for kern_data_resid == 0 instead; there
2714                          * may be several Data-In PDUs for the final
2715                          * call to cfiscsi_datamove(), and we want
2716                          * to set the F flag only on the last of them.
2717                          */
2718                         buffer_offset += response->ip_data_len;
2719                         if (buffer_offset == io->scsiio.kern_total_len ||
2720                             buffer_offset == expected_len)
2721                                 bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2722                         cfiscsi_pdu_queue(response);
2723                         response = NULL;
2724                         bhsdi = NULL;
2725                 }
2726         }
2727         if (response != NULL) {
2728                 buffer_offset += response->ip_data_len;
2729                 if (buffer_offset == io->scsiio.kern_total_len ||
2730                     buffer_offset == expected_len)
2731                         bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2732                 KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2733                 cfiscsi_pdu_queue(response);
2734         }
2735
2736         io->scsiio.be_move_done(io);
2737 }
2738
2739 static void
2740 cfiscsi_datamove_out(union ctl_io *io)
2741 {
2742         struct cfiscsi_session *cs;
2743         struct icl_pdu *request, *response;
2744         const struct iscsi_bhs_scsi_command *bhssc;
2745         struct iscsi_bhs_r2t *bhsr2t;
2746         struct cfiscsi_data_wait *cdw;
2747         uint32_t target_transfer_tag;
2748         bool done;
2749
2750         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2751         cs = PDU_SESSION(request);
2752
2753         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2754         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2755             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2756             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2757
2758         /*
2759          * We need to record it so that we can properly report
2760          * underflow/underflow.
2761          */
2762         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2763
2764         /*
2765          * We hadn't received anything during this datamove yet.
2766          */
2767         io->scsiio.ext_data_filled = 0;
2768
2769         target_transfer_tag =
2770             atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2771
2772 #if 0
2773         CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2774             "task tag 0x%x, target transfer tag 0x%x",
2775             bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2776 #endif
2777         cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
2778         if (cdw == NULL) {
2779                 CFISCSI_SESSION_WARN(cs, "failed to "
2780                     "allocate memory; dropping connection");
2781                 ctl_set_busy(&io->scsiio);
2782                 io->scsiio.be_move_done(io);
2783                 cfiscsi_session_terminate(cs);
2784                 return;
2785         }
2786         cdw->cdw_ctl_io = io;
2787         cdw->cdw_target_transfer_tag = target_transfer_tag;
2788         cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2789
2790         if (cs->cs_immediate_data && io->scsiio.kern_rel_offset <
2791             icl_pdu_data_segment_length(request)) {
2792                 done = cfiscsi_handle_data_segment(request, cdw);
2793                 if (done) {
2794                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2795                         io->scsiio.be_move_done(io);
2796                         return;
2797                 }
2798         }
2799
2800         CFISCSI_SESSION_LOCK(cs);
2801         TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2802         CFISCSI_SESSION_UNLOCK(cs);
2803
2804         /*
2805          * XXX: We should limit the number of outstanding R2T PDUs
2806          *      per task to MaxOutstandingR2T.
2807          */
2808         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2809         if (response == NULL) {
2810                 CFISCSI_SESSION_WARN(cs, "failed to "
2811                     "allocate memory; dropping connection");
2812                 ctl_set_busy(&io->scsiio);
2813                 io->scsiio.be_move_done(io);
2814                 cfiscsi_session_terminate(cs);
2815                 return;
2816         }
2817         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2818         bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2819         bhsr2t->bhsr2t_flags = 0x80;
2820         bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2821         bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2822         bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2823         /*
2824          * XXX: Here we assume that cfiscsi_datamove() won't ever
2825          *      be running concurrently on several CPUs for a given
2826          *      command.
2827          */
2828         bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2829         PDU_R2TSN(request)++;
2830         /*
2831          * This is the offset within the current SCSI command;
2832          * i.e. for the first call of datamove(), it will be 0,
2833          * and for subsequent ones it will be the sum of lengths
2834          * of previous ones.
2835          *
2836          * The ext_data_filled is to account for unsolicited
2837          * (immediate) data that might have already arrived.
2838          */
2839         bhsr2t->bhsr2t_buffer_offset =
2840             htonl(io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled);
2841         /*
2842          * This is the total length (sum of S/G lengths) this call
2843          * to cfiscsi_datamove() is supposed to handle.
2844          *
2845          * XXX: Limit it to MaxBurstLength.
2846          */
2847         bhsr2t->bhsr2t_desired_data_transfer_length =
2848             htonl(io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
2849         cfiscsi_pdu_queue(response);
2850 }
2851
2852 static void
2853 cfiscsi_datamove(union ctl_io *io)
2854 {
2855
2856         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2857                 cfiscsi_datamove_in(io);
2858         else
2859                 cfiscsi_datamove_out(io);
2860 }
2861
2862 static void
2863 cfiscsi_scsi_command_done(union ctl_io *io)
2864 {
2865         struct icl_pdu *request, *response;
2866         struct iscsi_bhs_scsi_command *bhssc;
2867         struct iscsi_bhs_scsi_response *bhssr;
2868 #ifdef DIAGNOSTIC
2869         struct cfiscsi_data_wait *cdw;
2870 #endif
2871         struct cfiscsi_session *cs;
2872         uint16_t sense_length;
2873
2874         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2875         cs = PDU_SESSION(request);
2876         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2877         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2878             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2879             ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2880
2881         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2882         //    bhssc->bhssc_initiator_task_tag);
2883
2884 #ifdef DIAGNOSTIC
2885         CFISCSI_SESSION_LOCK(cs);
2886         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2887                 KASSERT(bhssc->bhssc_initiator_task_tag !=
2888                     cdw->cdw_initiator_task_tag, ("dangling cdw"));
2889         CFISCSI_SESSION_UNLOCK(cs);
2890 #endif
2891
2892         /*
2893          * Do not return status for aborted commands.
2894          * There are exceptions, but none supported by CTL yet.
2895          */
2896         if (io->io_hdr.status == CTL_CMD_ABORTED) {
2897                 ctl_free_io(io);
2898                 icl_pdu_free(request);
2899                 return;
2900         }
2901
2902         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2903         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2904         bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2905         bhssr->bhssr_flags = 0x80;
2906         /*
2907          * XXX: We don't deal with bidirectional under/overflows;
2908          *      does anything actually support those?
2909          */
2910         if (PDU_TOTAL_TRANSFER_LEN(request) <
2911             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2912                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2913                 bhssr->bhssr_residual_count =
2914                     htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2915                     PDU_TOTAL_TRANSFER_LEN(request));
2916                 //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2917                 //    ntohl(bhssr->bhssr_residual_count));
2918         } else if (PDU_TOTAL_TRANSFER_LEN(request) >
2919             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2920                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2921                 bhssr->bhssr_residual_count =
2922                     htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2923                     ntohl(bhssc->bhssc_expected_data_transfer_length));
2924                 //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2925                 //    ntohl(bhssr->bhssr_residual_count));
2926         }
2927         bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2928         bhssr->bhssr_status = io->scsiio.scsi_status;
2929         bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2930         bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2931
2932         if (io->scsiio.sense_len > 0) {
2933 #if 0
2934                 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2935                     io->scsiio.sense_len);
2936 #endif
2937                 sense_length = htons(io->scsiio.sense_len);
2938                 icl_pdu_append_data(response,
2939                     &sense_length, sizeof(sense_length), M_WAITOK);
2940                 icl_pdu_append_data(response,
2941                     &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2942         }
2943
2944         ctl_free_io(io);
2945         icl_pdu_free(request);
2946         cfiscsi_pdu_queue(response);
2947 }
2948
2949 static void
2950 cfiscsi_task_management_done(union ctl_io *io)
2951 {
2952         struct icl_pdu *request, *response;
2953         struct iscsi_bhs_task_management_request *bhstmr;
2954         struct iscsi_bhs_task_management_response *bhstmr2;
2955         struct cfiscsi_data_wait *cdw, *tmpcdw;
2956         struct cfiscsi_session *cs;
2957
2958         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2959         cs = PDU_SESSION(request);
2960         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2961         KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2962             ISCSI_BHS_OPCODE_TASK_REQUEST,
2963             ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2964
2965 #if 0
2966         CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2967             bhstmr->bhstmr_initiator_task_tag,
2968             bhstmr->bhstmr_referenced_task_tag);
2969 #endif
2970
2971         if ((bhstmr->bhstmr_function & ~0x80) ==
2972             BHSTMR_FUNCTION_ABORT_TASK) {
2973                 /*
2974                  * Make sure we no longer wait for Data-Out for this command.
2975                  */
2976                 CFISCSI_SESSION_LOCK(cs);
2977                 TAILQ_FOREACH_SAFE(cdw,
2978                     &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2979                         if (bhstmr->bhstmr_referenced_task_tag !=
2980                             cdw->cdw_initiator_task_tag)
2981                                 continue;
2982
2983 #if 0
2984                         CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2985                             "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2986 #endif
2987                         TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2988                             cdw, cdw_next);
2989                         cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2990                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2991                 }
2992                 CFISCSI_SESSION_UNLOCK(cs);
2993         }
2994
2995         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2996         bhstmr2 = (struct iscsi_bhs_task_management_response *)
2997             response->ip_bhs;
2998         bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2999         bhstmr2->bhstmr_flags = 0x80;
3000         if (io->io_hdr.status == CTL_SUCCESS) {
3001                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
3002         } else {
3003                 /*
3004                  * XXX: How to figure out what exactly went wrong?  iSCSI spec
3005                  *      expects us to provide detailed error, e.g. "Task does
3006                  *      not exist" or "LUN does not exist".
3007                  */
3008                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED");
3009                 bhstmr2->bhstmr_response =
3010                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
3011         }
3012         bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
3013
3014         ctl_free_io(io);
3015         icl_pdu_free(request);
3016         cfiscsi_pdu_queue(response);
3017 }
3018
3019 static void
3020 cfiscsi_done(union ctl_io *io)
3021 {
3022         struct icl_pdu *request;
3023         struct cfiscsi_session *cs;
3024
3025         KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
3026                 ("invalid CTL status %#x", io->io_hdr.status));
3027
3028         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
3029         if (request == NULL) {
3030                 /*
3031                  * Implicit task termination has just completed; nothing to do.
3032                  */
3033                 return;
3034         }
3035
3036         cs = PDU_SESSION(request);
3037         refcount_release(&cs->cs_outstanding_ctl_pdus);
3038
3039         switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3040         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3041                 cfiscsi_scsi_command_done(io);
3042                 break;
3043         case ISCSI_BHS_OPCODE_TASK_REQUEST:
3044                 cfiscsi_task_management_done(io);
3045                 break;
3046         default:
3047                 panic("cfiscsi_done called with wrong opcode 0x%x",
3048                     request->ip_bhs->bhs_opcode);
3049         }
3050 }