sys/cam/ctl/ctl_frontend_iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 /*
  33  * CTL frontend for the iSCSI protocol.
  34  */
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include <sys/param.h>
  40 #include <sys/capability.h>
  41 #include <sys/condvar.h>
  42 #include <sys/file.h>
  43 #include <sys/kernel.h>
  44 #include <sys/kthread.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mutex.h>
  49 #include <sys/queue.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/systm.h>
  53 #include <sys/uio.h>
  54 #include <sys/unistd.h>
  55 #include <vm/uma.h>
  56
  57 #include <cam/scsi/scsi_all.h>
  58 #include <cam/scsi/scsi_da.h>
  59 #include <cam/ctl/ctl_io.h>
  60 #include <cam/ctl/ctl.h>
  61 #include <cam/ctl/ctl_backend.h>
  62 #include <cam/ctl/ctl_error.h>
  63 #include <cam/ctl/ctl_frontend.h>
  64 #include <cam/ctl/ctl_frontend_internal.h>
  65 #include <cam/ctl/ctl_debug.h>
  66 #include <cam/ctl/ctl_ha.h>
  67 #include <cam/ctl/ctl_ioctl.h>
  68 #include <cam/ctl/ctl_private.h>
  69
  70 #include "../../dev/iscsi/icl.h"
  71 #include "../../dev/iscsi/iscsi_proto.h"
  72 #include "ctl_frontend_iscsi.h"
  73
  74 #ifdef ICL_KERNEL_PROXY
  75 #include <sys/socketvar.h>
  76 #endif
  77
  78 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
  79 static uma_zone_t cfiscsi_data_wait_zone;
  80
  81 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
  82     "CAM Target Layer iSCSI Frontend");
  83 static int debug = 3;
  84 TUNABLE_INT("kern.cam.ctl.iscsi.debug", &debug);
  85 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RW,
  86     &debug, 1, "Enable debug messages");
  87 static int ping_timeout = 5;
  88 TUNABLE_INT("kern.cam.ctl.iscsi.ping_timeout", &ping_timeout);
  89 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RW,
  90     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
  91 static int login_timeout = 60;
  92 TUNABLE_INT("kern.cam.ctl.iscsi.login_timeout", &login_timeout);
  93 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RW,
  94     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
  95 static int maxcmdsn_delta = 256;
  96 TUNABLE_INT("kern.cam.ctl.iscsi.maxcmdsn_delta", &maxcmdsn_delta);
  97 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RW,
  98     &maxcmdsn_delta, 256, "Number of commands the initiator can send "
  99     "without confirmation");
 100
 101 #define CFISCSI_DEBUG(X, ...)                                   \
 102         if (debug > 1) {                                        \
 103                 printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 104         } while (0)
 105
 106 #define CFISCSI_WARN(X, ...)                                    \
 107         if (debug > 0) {                                        \
 108                 printf("WARNING: %s: " X "\n",                  \
 109                     __func__, ## __VA_ARGS__);                  \
 110         } while (0)
 111
 112 #define CFISCSI_SESSION_DEBUG(S, X, ...)                        \
 113         if (debug > 1) {                                        \
 114                 printf("%s: %s (%s): " X "\n",                  \
 115                     __func__, S->cs_initiator_addr,             \
 116                     S->cs_initiator_name, ## __VA_ARGS__);      \
 117         } while (0)
 118
 119 #define CFISCSI_SESSION_WARN(S, X, ...)                         \
 120         if (debug > 0) {                                        \
 121                 printf("WARNING: %s (%s): " X "\n",             \
 122                     S->cs_initiator_addr,                       \
 123                     S->cs_initiator_name, ## __VA_ARGS__);      \
 124         } while (0)
 125
 126 #define CFISCSI_SESSION_LOCK(X)         mtx_lock(&X->cs_lock)
 127 #define CFISCSI_SESSION_UNLOCK(X)       mtx_unlock(&X->cs_lock)
 128 #define CFISCSI_SESSION_LOCK_ASSERT(X)  mtx_assert(&X->cs_lock, MA_OWNED)
 129
 130 #define CONN_SESSION(X)                 ((struct cfiscsi_session *)(X)->ic_prv0)
 131 #define PDU_SESSION(X)                  CONN_SESSION((X)->ip_conn)
 132 #define PDU_EXPDATASN(X)                (X)->ip_prv0
 133 #define PDU_TOTAL_TRANSFER_LEN(X)       (X)->ip_prv1
 134 #define PDU_R2TSN(X)                    (X)->ip_prv2
 135
 136 int             cfiscsi_init(void);
 137 static void     cfiscsi_online(void *arg);
 138 static void     cfiscsi_offline(void *arg);
 139 static int      cfiscsi_targ_enable(void *arg, struct ctl_id targ_id);
 140 static int      cfiscsi_targ_disable(void *arg, struct ctl_id targ_id);
 141 static int      cfiscsi_lun_enable(void *arg,
 142                     struct ctl_id target_id, int lun_id);
 143 static int      cfiscsi_lun_disable(void *arg,
 144                     struct ctl_id target_id, int lun_id);
 145 static int      cfiscsi_ioctl(struct cdev *dev,
 146                     u_long cmd, caddr_t addr, int flag, struct thread *td);
 147 static int      cfiscsi_devid(struct ctl_scsiio *ctsio, int alloc_len);
 148 static void     cfiscsi_datamove(union ctl_io *io);
 149 static void     cfiscsi_done(union ctl_io *io);
 150 static uint32_t cfiscsi_map_lun(void *arg, uint32_t lun);
 151 static bool     cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
 152 static void     cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
 153 static void     cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
 154 static void     cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
 155 static void     cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
 156 static void     cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
 157 static void     cfiscsi_session_terminate(struct cfiscsi_session *cs);
 158 static struct cfiscsi_target    *cfiscsi_target_find(struct cfiscsi_softc
 159                     *softc, const char *name);
 160 static void     cfiscsi_target_release(struct cfiscsi_target *ct);
 161 static void     cfiscsi_session_delete(struct cfiscsi_session *cs);
 162
 163 static struct cfiscsi_softc cfiscsi_softc;
 164 extern struct ctl_softc *control_softc;
 165
 166 static int cfiscsi_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 167
 168 static moduledata_t cfiscsi_moduledata = {
 169         "ctlcfiscsi",
 170         cfiscsi_module_event_handler,
 171         NULL
 172 };
 173
 174 DECLARE_MODULE(ctlcfiscsi, cfiscsi_moduledata, SI_SUB_CONFIGURE, SI_ORDER_FOURTH);
 175 MODULE_VERSION(ctlcfiscsi, 1);
 176 MODULE_DEPEND(ctlcfiscsi, ctl, 1, 1, 1);
 177 MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1);
 178
 179 static struct icl_pdu *
 180 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
 181 {
 182
 183         return (icl_pdu_new_bhs(request->ip_conn, flags));
 184 }
 185
 186 static bool
 187 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
 188 {
 189         const struct iscsi_bhs_scsi_command *bhssc;
 190         struct cfiscsi_session *cs;
 191         uint32_t cmdsn, expstatsn;
 192
 193         cs = PDU_SESSION(request);
 194
 195         /*
 196          * Every incoming PDU - not just NOP-Out - resets the ping timer.
 197          * The purpose of the timeout is to reset the connection when it stalls;
 198          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 199          * in some queue.
 200          *
 201          * XXX: Locking?
 202          */
 203         cs->cs_timeout = 0;
 204
 205         /*
 206          * Data-Out PDUs don't contain CmdSN.
 207          */
 208         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 209             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 210                 return (false);
 211
 212         /*
 213          * We're only using fields common for all the request
 214          * (initiator -> target) PDUs.
 215          */
 216         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
 217         cmdsn = ntohl(bhssc->bhssc_cmdsn);
 218         expstatsn = ntohl(bhssc->bhssc_expstatsn);
 219
 220         CFISCSI_SESSION_LOCK(cs);
 221 #if 0
 222         if (expstatsn != cs->cs_statsn) {
 223                 CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
 224                     "while current StatSN is %d", expstatsn,
 225                     cs->cs_statsn);
 226         }
 227 #endif
 228
 229         /*
 230          * The target MUST silently ignore any non-immediate command outside
 231          * of this range.
 232          */
 233         if (cmdsn < cs->cs_cmdsn || cmdsn > cs->cs_cmdsn + maxcmdsn_delta) {
 234                 CFISCSI_SESSION_UNLOCK(cs);
 235                 CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %d, "
 236                     "while expected CmdSN was %d", cmdsn, cs->cs_cmdsn);
 237                 return (true);
 238         }
 239
 240         if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 241                 cs->cs_cmdsn++;
 242
 243         CFISCSI_SESSION_UNLOCK(cs);
 244
 245         return (false);
 246 }
 247
 248 static void
 249 cfiscsi_pdu_handle(struct icl_pdu *request)
 250 {
 251         struct cfiscsi_session *cs;
 252         bool ignore;
 253
 254         cs = PDU_SESSION(request);
 255
 256         ignore = cfiscsi_pdu_update_cmdsn(request);
 257         if (ignore) {
 258                 icl_pdu_free(request);
 259                 return;
 260         }
 261
 262         /*
 263          * Handle the PDU; this includes e.g. receiving the remaining
 264          * part of PDU and submitting the SCSI command to CTL
 265          * or queueing a reply.  The handling routine is responsible
 266          * for freeing the PDU when it's no longer needed.
 267          */
 268         switch (request->ip_bhs->bhs_opcode &
 269             ~ISCSI_BHS_OPCODE_IMMEDIATE) {
 270         case ISCSI_BHS_OPCODE_NOP_OUT:
 271                 cfiscsi_pdu_handle_nop_out(request);
 272                 break;
 273         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
 274                 cfiscsi_pdu_handle_scsi_command(request);
 275                 break;
 276         case ISCSI_BHS_OPCODE_TASK_REQUEST:
 277                 cfiscsi_pdu_handle_task_request(request);
 278                 break;
 279         case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
 280                 cfiscsi_pdu_handle_data_out(request);
 281                 break;
 282         case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
 283                 cfiscsi_pdu_handle_logout_request(request);
 284                 break;
 285         default:
 286                 CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
 287                     "opcode 0x%x; dropping connection",
 288                     request->ip_bhs->bhs_opcode);
 289                 icl_pdu_free(request);
 290                 cfiscsi_session_terminate(cs);
 291         }
 292
 293 }
 294
 295 static void
 296 cfiscsi_receive_callback(struct icl_pdu *request)
 297 {
 298         struct cfiscsi_session *cs;
 299
 300         cs = PDU_SESSION(request);
 301
 302 #ifdef ICL_KERNEL_PROXY
 303         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 304                 if (cs->cs_login_pdu == NULL)
 305                         cs->cs_login_pdu = request;
 306                 else
 307                         icl_pdu_free(request);
 308                 cv_signal(&cs->cs_login_cv);
 309                 return;
 310         }
 311 #endif
 312
 313         cfiscsi_pdu_handle(request);
 314 }
 315
 316 static void
 317 cfiscsi_error_callback(struct icl_conn *ic)
 318 {
 319         struct cfiscsi_session *cs;
 320
 321         cs = CONN_SESSION(ic);
 322
 323         CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
 324         cfiscsi_session_terminate(cs);
 325 }
 326
 327 static int
 328 cfiscsi_pdu_prepare(struct icl_pdu *response)
 329 {
 330         struct cfiscsi_session *cs;
 331         struct iscsi_bhs_scsi_response *bhssr;
 332         bool advance_statsn = true;
 333
 334         cs = PDU_SESSION(response);
 335
 336         CFISCSI_SESSION_LOCK_ASSERT(cs);
 337
 338         /*
 339          * We're only using fields common for all the response
 340          * (target -> initiator) PDUs.
 341          */
 342         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 343
 344         /*
 345          * 10.8.3: "The StatSN for this connection is not advanced
 346          * after this PDU is sent."
 347          */
 348         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
 349                 advance_statsn = false;
 350
 351         /*
 352          * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
 353          * StatSN for the connection is not advanced after this PDU is sent."
 354          */
 355         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
 356             bhssr->bhssr_initiator_task_tag == 0xffffffff)
 357                 advance_statsn = false;
 358
 359         /*
 360          * See the comment below - StatSN is not meaningful and must
 361          * not be advanced.
 362          */
 363         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN)
 364                 advance_statsn = false;
 365
 366         /*
 367          * 10.7.3: "The fields StatSN, Status, and Residual Count
 368          * only have meaningful content if the S bit is set to 1."
 369          */
 370         if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN)
 371                 bhssr->bhssr_statsn = htonl(cs->cs_statsn);
 372         bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
 373         bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn + maxcmdsn_delta);
 374
 375         if (advance_statsn)
 376                 cs->cs_statsn++;
 377
 378         return (0);
 379 }
 380
 381 static void
 382 cfiscsi_pdu_queue(struct icl_pdu *response)
 383 {
 384         struct cfiscsi_session *cs;
 385
 386         cs = PDU_SESSION(response);
 387
 388         CFISCSI_SESSION_LOCK(cs);
 389         cfiscsi_pdu_prepare(response);
 390         icl_pdu_queue(response);
 391         CFISCSI_SESSION_UNLOCK(cs);
 392 }
 393
 394 static uint32_t
 395 cfiscsi_decode_lun(uint64_t encoded)
 396 {
 397         uint8_t lun[8];
 398         uint32_t result;
 399
 400         /*
 401          * The LUN field in iSCSI PDUs may look like an ordinary 64 bit number,
 402          * but is in fact an evil, multidimensional structure defined
 403          * in SCSI Architecture Model 5 (SAM-5), section 4.6.
 404          */
 405         memcpy(lun, &encoded, sizeof(lun));
 406         switch (lun[0] & 0xC0) {
 407         case 0x00:
 408                 if ((lun[0] & 0x3f) != 0 || lun[2] != 0 || lun[3] != 0 ||
 409                     lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) {
 410                         CFISCSI_WARN("malformed LUN "
 411                             "(peripheral device addressing method): 0x%jx",
 412                             (uintmax_t)encoded);
 413                         result = 0xffffffff;
 414                         break;
 415                 }
 416                 result = lun[1];
 417                 break;
 418         case 0x40:
 419                 if (lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 ||
 420                     lun[6] != 0 || lun[7] != 0) {
 421                         CFISCSI_WARN("malformed LUN "
 422                             "(flat address space addressing method): 0x%jx",
 423                             (uintmax_t)encoded);
 424                         result = 0xffffffff;
 425                         break;
 426                 }
 427                 result = ((lun[0] & 0x3f) << 8) + lun[1];
 428                 break;
 429         case 0xC0:
 430                 if (lun[0] != 0xD2 || lun[4] != 0 || lun[5] != 0 ||
 431                     lun[6] != 0 || lun[7] != 0) {
 432                         CFISCSI_WARN("malformed LUN (extended flat "
 433                             "address space addressing method): 0x%jx",
 434                             (uintmax_t)encoded);
 435                         result = 0xffffffff;
 436                         break;
 437                 }
 438                 result = (lun[1] << 16) + (lun[2] << 8) + lun[3];
 439         default:
 440                 CFISCSI_WARN("unsupported LUN format 0x%jx",
 441                     (uintmax_t)encoded);
 442                 result = 0xffffffff;
 443                 break;
 444         }
 445
 446         return (result);
 447 }
 448
 449 static void
 450 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
 451 {
 452         struct cfiscsi_session *cs;
 453         struct iscsi_bhs_nop_out *bhsno;
 454         struct iscsi_bhs_nop_in *bhsni;
 455         struct icl_pdu *response;
 456         void *data = NULL;
 457         size_t datasize;
 458         int error;
 459
 460         cs = PDU_SESSION(request);
 461         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 462
 463         if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
 464                 /*
 465                  * Nothing to do, iscsi_pdu_update_statsn() already
 466                  * zeroed the timeout.
 467                  */
 468                 icl_pdu_free(request);
 469                 return;
 470         }
 471
 472         datasize = icl_pdu_data_segment_length(request);
 473         if (datasize > 0) {
 474                 data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
 475                 if (data == NULL) {
 476                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 477                             "dropping connection");
 478                         icl_pdu_free(request);
 479                         cfiscsi_session_terminate(cs);
 480                         return;
 481                 }
 482                 icl_pdu_get_data(request, 0, data, datasize);
 483         }
 484
 485         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 486         if (response == NULL) {
 487                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 488                     "droppping connection");
 489                 free(data, M_CFISCSI);
 490                 icl_pdu_free(request);
 491                 cfiscsi_session_terminate(cs);
 492                 return;
 493         }
 494         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 495         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
 496         bhsni->bhsni_flags = 0x80;
 497         bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
 498         bhsni->bhsni_target_transfer_tag = 0xffffffff;
 499         if (datasize > 0) {
 500                 error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
 501                 if (error != 0) {
 502                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 503                             "dropping connection");
 504                         free(data, M_CFISCSI);
 505                         icl_pdu_free(request);
 506                         icl_pdu_free(response);
 507                         cfiscsi_session_terminate(cs);
 508                         return;
 509                 }
 510                 free(data, M_CFISCSI);
 511         }
 512
 513         icl_pdu_free(request);
 514         cfiscsi_pdu_queue(response);
 515 }
 516
 517 static void
 518 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
 519 {
 520         struct iscsi_bhs_scsi_command *bhssc;
 521         struct cfiscsi_session *cs;
 522         union ctl_io *io;
 523         int error;
 524
 525         cs = PDU_SESSION(request);
 526         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 527         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
 528         //    bhssc->bhssc_initiator_task_tag);
 529
 530         if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
 531                 CFISCSI_SESSION_WARN(cs, "unsolicited data with "
 532                     "ImmediateData=No; dropping connection");
 533                 icl_pdu_free(request);
 534                 cfiscsi_session_terminate(cs);
 535                 return;
 536         }
 537         io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
 538         if (io == NULL) {
 539                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io; "
 540                     "dropping connection");
 541                 icl_pdu_free(request);
 542                 cfiscsi_session_terminate(cs);
 543                 return;
 544         }
 545         ctl_zero_io(io);
 546         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 547         io->io_hdr.io_type = CTL_IO_SCSI;
 548         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 549         io->io_hdr.nexus.targ_port = cs->cs_target->ct_softc->fe.targ_port;
 550         io->io_hdr.nexus.targ_target.id = 0;
 551         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhssc->bhssc_lun);
 552         io->io_hdr.nexus.lun_map_fn = cfiscsi_map_lun;
 553         io->io_hdr.nexus.lun_map_arg = cs;
 554         io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
 555         switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
 556         case BHSSC_FLAGS_ATTR_UNTAGGED:
 557                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 558                 break;
 559         case BHSSC_FLAGS_ATTR_SIMPLE:
 560                 io->scsiio.tag_type = CTL_TAG_SIMPLE;
 561                 break;
 562         case BHSSC_FLAGS_ATTR_ORDERED:
 563                 io->scsiio.tag_type = CTL_TAG_ORDERED;
 564                 break;
 565         case BHSSC_FLAGS_ATTR_HOQ:
 566                 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
 567                 break;
 568         case BHSSC_FLAGS_ATTR_ACA:
 569                 io->scsiio.tag_type = CTL_TAG_ACA;
 570                 break;
 571         default:
 572                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 573                 CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
 574                     bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
 575                 break;
 576         }
 577         io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
 578         memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
 579         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 580         error = ctl_queue(io);
 581         if (error != CTL_RETVAL_COMPLETE) {
 582                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 583                     "dropping connection", error);
 584                 ctl_free_io(io);
 585                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 586                 icl_pdu_free(request);
 587                 cfiscsi_session_terminate(cs);
 588         }
 589 }
 590
 591 static void
 592 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
 593 {
 594         struct iscsi_bhs_task_management_request *bhstmr;
 595         struct iscsi_bhs_task_management_response *bhstmr2;
 596         struct icl_pdu *response;
 597         struct cfiscsi_session *cs;
 598         union ctl_io *io;
 599         int error;
 600
 601         cs = PDU_SESSION(request);
 602         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
 603         io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
 604         if (io == NULL) {
 605                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io;"
 606                     "dropping connection");
 607                 icl_pdu_free(request);
 608                 cfiscsi_session_terminate(cs);
 609                 return;
 610         }
 611         ctl_zero_io(io);
 612         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 613         io->io_hdr.io_type = CTL_IO_TASK;
 614         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 615         io->io_hdr.nexus.targ_port = cs->cs_target->ct_softc->fe.targ_port;
 616         io->io_hdr.nexus.targ_target.id = 0;
 617         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhstmr->bhstmr_lun);
 618         io->io_hdr.nexus.lun_map_fn = cfiscsi_map_lun;
 619         io->io_hdr.nexus.lun_map_arg = cs;
 620         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
 621
 622         switch (bhstmr->bhstmr_function & ~0x80) {
 623         case BHSTMR_FUNCTION_ABORT_TASK:
 624 #if 0
 625                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
 626 #endif
 627                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
 628                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
 629                 break;
 630         case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
 631 #if 0
 632                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
 633 #endif
 634                 io->taskio.task_action = CTL_TASK_LUN_RESET;
 635                 break;
 636         case BHSTMR_FUNCTION_TARGET_COLD_RESET:
 637 #if 0
 638                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
 639 #endif
 640                 io->taskio.task_action = CTL_TASK_BUS_RESET;
 641                 break;
 642         default:
 643                 CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
 644                     bhstmr->bhstmr_function & ~0x80);
 645                 ctl_free_io(io);
 646
 647                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 648                 if (response == NULL) {
 649                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 650                             "dropping connection");
 651                         icl_pdu_free(request);
 652                         cfiscsi_session_terminate(cs);
 653                         return;
 654                 }
 655                 bhstmr2 = (struct iscsi_bhs_task_management_response *)
 656                     response->ip_bhs;
 657                 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
 658                 bhstmr2->bhstmr_flags = 0x80;
 659                 bhstmr2->bhstmr_response =
 660                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
 661                 bhstmr2->bhstmr_initiator_task_tag =
 662                     bhstmr->bhstmr_initiator_task_tag;
 663                 icl_pdu_free(request);
 664                 cfiscsi_pdu_queue(response);
 665                 return;
 666         }
 667
 668         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 669         error = ctl_queue(io);
 670         if (error != CTL_RETVAL_COMPLETE) {
 671                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 672                     "dropping connection", error);
 673                 ctl_free_io(io);
 674                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 675                 icl_pdu_free(request);
 676                 cfiscsi_session_terminate(cs);
 677         }
 678 }
 679
 680 static bool
 681 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
 682 {
 683         struct iscsi_bhs_data_out *bhsdo;
 684         struct cfiscsi_session *cs;
 685         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
 686         size_t copy_len, len, off, buffer_offset;
 687         int ctl_sg_count;
 688         union ctl_io *io;
 689
 690         cs = PDU_SESSION(request);
 691
 692         KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 693             ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
 694             (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 695             ISCSI_BHS_OPCODE_SCSI_COMMAND,
 696             ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
 697
 698         /*
 699          * We're only using fields common for Data Out and SCSI Command PDUs.
 700          */
 701         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 702
 703         io = cdw->cdw_ctl_io;
 704         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 705             ("CTL_FLAG_DATA_IN"));
 706
 707 #if 0
 708         CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
 709             request->ip_data_len, io->scsiio.kern_total_len);
 710 #endif
 711
 712         if (io->scsiio.kern_sg_entries > 0) {
 713                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
 714                 ctl_sg_count = io->scsiio.kern_sg_entries;
 715         } else {
 716                 ctl_sglist = &ctl_sg_entry;
 717                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
 718                 ctl_sglist->len = io->scsiio.kern_data_len;
 719                 ctl_sg_count = 1;
 720         }
 721
 722         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 723             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 724                 buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
 725         else
 726                 buffer_offset = 0;
 727
 728         /*
 729          * Make sure the offset, as sent by the initiator, matches the offset
 730          * we're supposed to be at in the scatter-gather list.
 731          */
 732         if (buffer_offset != io->scsiio.ext_data_filled) {
 733                 CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
 734                     "expected %zd", buffer_offset,
 735                     (size_t)io->scsiio.ext_data_filled);
 736                 cfiscsi_session_terminate(cs);
 737                 return (true);
 738         }
 739
 740         /*
 741          * This is the offset within the PDU data segment, as opposed
 742          * to buffer_offset, which is the offset within the task (SCSI
 743          * command).
 744          */
 745         off = 0;
 746         len = icl_pdu_data_segment_length(request);
 747
 748         /*
 749          * Iterate over the scatter/gather segments, filling them with data
 750          * from the PDU data segment.  Note that this can get called multiple
 751          * times for one SCSI command; the cdw structure holds state for the
 752          * scatter/gather list.
 753          */
 754         for (;;) {
 755                 KASSERT(cdw->cdw_sg_index < ctl_sg_count,
 756                     ("cdw->cdw_sg_index >= ctl_sg_count"));
 757                 if (cdw->cdw_sg_len == 0) {
 758                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
 759                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
 760                 }
 761                 KASSERT(off <= len, ("len > off"));
 762                 copy_len = len - off;
 763                 if (copy_len > cdw->cdw_sg_len)
 764                         copy_len = cdw->cdw_sg_len;
 765
 766                 icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
 767                 cdw->cdw_sg_addr += copy_len;
 768                 cdw->cdw_sg_len -= copy_len;
 769                 off += copy_len;
 770                 io->scsiio.ext_data_filled += copy_len;
 771
 772                 if (cdw->cdw_sg_len == 0) {
 773                         /*
 774                          * End of current segment.
 775                          */
 776                         if (cdw->cdw_sg_index == ctl_sg_count - 1) {
 777                                 /*
 778                                  * Last segment in scatter/gather list.
 779                                  */
 780                                 break;
 781                         }
 782                         cdw->cdw_sg_index++;
 783                 }
 784
 785                 if (off == len) {
 786                         /*
 787                          * End of PDU payload.
 788                          */
 789                         break;
 790                 }
 791         }
 792
 793         if (len > off) {
 794                 CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
 795                     "expected %zd", icl_pdu_data_segment_length(request), off);
 796                 cfiscsi_session_terminate(cs);
 797                 return (true);
 798         }
 799
 800         if (bhsdo->bhsdo_flags & BHSDO_FLAGS_F ||
 801             io->scsiio.ext_data_filled == io->scsiio.kern_total_len) {
 802                 if ((bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
 803                         CFISCSI_SESSION_WARN(cs, "got the final packet without "
 804                             "the F flag; flags = 0x%x; dropping connection",
 805                             bhsdo->bhsdo_flags);
 806                         cfiscsi_session_terminate(cs);
 807                         return (true);
 808                 }
 809
 810                 if (io->scsiio.ext_data_filled != io->scsiio.kern_total_len) {
 811                         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 812                             ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 813                                 CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
 814                                     "transmitted size was %zd bytes instead of %d; "
 815                                     "dropping connection",
 816                                     (size_t)io->scsiio.ext_data_filled,
 817                                     io->scsiio.kern_total_len);
 818                                 cfiscsi_session_terminate(cs);
 819                                 return (true);
 820                         } else {
 821                                 /*
 822                                  * For SCSI Command PDU, this just means we need to
 823                                  * solicit more data by sending R2T.
 824                                  */
 825                                 return (false);
 826                         }
 827                 }
 828 #if 0
 829                 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
 830                     "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
 831 #endif
 832
 833                 return (true);
 834         }
 835
 836         return (false);
 837 }
 838
 839 static void
 840 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
 841 {
 842         struct iscsi_bhs_data_out *bhsdo;
 843         struct cfiscsi_session *cs;
 844         struct cfiscsi_data_wait *cdw = NULL;
 845         union ctl_io *io;
 846         bool done;
 847
 848         cs = PDU_SESSION(request);
 849         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 850
 851         CFISCSI_SESSION_LOCK(cs);
 852         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
 853 #if 0
 854                 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
 855                     "ttt 0x%x, itt 0x%x",
 856                     bhsdo->bhsdo_target_transfer_tag,
 857                     bhsdo->bhsdo_initiator_task_tag,
 858                     cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
 859 #endif
 860                 if (bhsdo->bhsdo_target_transfer_tag ==
 861                     cdw->cdw_target_transfer_tag)
 862                         break;
 863         }
 864         CFISCSI_SESSION_UNLOCK(cs);
 865         if (cdw == NULL) {
 866                 CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
 867                     "0x%x, not found; dropping connection",
 868                     bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
 869                 icl_pdu_free(request);
 870                 cfiscsi_session_terminate(cs);
 871                 return;
 872         }
 873
 874         io = cdw->cdw_ctl_io;
 875         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 876             ("CTL_FLAG_DATA_IN"));
 877
 878         done = cfiscsi_handle_data_segment(request, cdw);
 879         if (done) {
 880                 CFISCSI_SESSION_LOCK(cs);
 881                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
 882                 CFISCSI_SESSION_UNLOCK(cs);
 883                 uma_zfree(cfiscsi_data_wait_zone, cdw);
 884                 io->scsiio.be_move_done(io);
 885         }
 886
 887         icl_pdu_free(request);
 888 }
 889
 890 static void
 891 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
 892 {
 893         struct iscsi_bhs_logout_request *bhslr;
 894         struct iscsi_bhs_logout_response *bhslr2;
 895         struct icl_pdu *response;
 896         struct cfiscsi_session *cs;
 897
 898         cs = PDU_SESSION(request);
 899         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 900         switch (bhslr->bhslr_reason & 0x7f) {
 901         case BHSLR_REASON_CLOSE_SESSION:
 902         case BHSLR_REASON_CLOSE_CONNECTION:
 903                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 904                 if (response == NULL) {
 905                         CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
 906                         icl_pdu_free(request);
 907                         cfiscsi_session_terminate(cs);
 908                         return;
 909                 }
 910                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 911                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 912                 bhslr2->bhslr_flags = 0x80;
 913                 bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
 914                 bhslr2->bhslr_initiator_task_tag =
 915                     bhslr->bhslr_initiator_task_tag;
 916                 icl_pdu_free(request);
 917                 cfiscsi_pdu_queue(response);
 918                 cfiscsi_session_terminate(cs);
 919                 break;
 920         case BHSLR_REASON_REMOVE_FOR_RECOVERY:
 921                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 922                 if (response == NULL) {
 923                         CFISCSI_SESSION_WARN(cs,
 924                             "failed to allocate memory; dropping connection");
 925                         icl_pdu_free(request);
 926                         cfiscsi_session_terminate(cs);
 927                         return;
 928                 }
 929                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 930                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 931                 bhslr2->bhslr_flags = 0x80;
 932                 bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
 933                 bhslr2->bhslr_initiator_task_tag =
 934                     bhslr->bhslr_initiator_task_tag;
 935                 icl_pdu_free(request);
 936                 cfiscsi_pdu_queue(response);
 937                 break;
 938         default:
 939                 CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
 940                     bhslr->bhslr_reason);
 941                 icl_pdu_free(request);
 942                 cfiscsi_session_terminate(cs);
 943                 break;
 944         }
 945 }
 946
 947 static void
 948 cfiscsi_callout(void *context)
 949 {
 950         struct icl_pdu *cp;
 951         struct iscsi_bhs_nop_in *bhsni;
 952         struct cfiscsi_session *cs;
 953
 954         cs = context;
 955
 956         if (cs->cs_terminating)
 957                 return;
 958
 959         callout_schedule(&cs->cs_callout, 1 * hz);
 960
 961         CFISCSI_SESSION_LOCK(cs);
 962         cs->cs_timeout++;
 963         CFISCSI_SESSION_UNLOCK(cs);
 964
 965 #ifdef ICL_KERNEL_PROXY
 966         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 967                 if (cs->cs_timeout > login_timeout) {
 968                         CFISCSI_SESSION_WARN(cs, "login timed out after "
 969                             "%d seconds; dropping connection", cs->cs_timeout);
 970                         cfiscsi_session_terminate(cs);
 971                 }
 972                 return;
 973         }
 974 #endif
 975
 976         if (cs->cs_timeout >= ping_timeout) {
 977                 CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
 978                     "dropping connection",  ping_timeout);
 979                 cfiscsi_session_terminate(cs);
 980                 return;
 981         }
 982
 983         /*
 984          * If the ping was reset less than one second ago - which means
 985          * that we've received some PDU during the last second - assume
 986          * the traffic flows correctly and don't bother sending a NOP-Out.
 987          *
 988          * (It's 2 - one for one second, and one for incrementing is_timeout
 989          * earlier in this routine.)
 990          */
 991         if (cs->cs_timeout < 2)
 992                 return;
 993
 994         cp = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
 995         if (cp == NULL) {
 996                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
 997                 return;
 998         }
 999         bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1000         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1001         bhsni->bhsni_flags = 0x80;
1002         bhsni->bhsni_initiator_task_tag = 0xffffffff;
1003
1004         cfiscsi_pdu_queue(cp);
1005 }
1006
1007 static void
1008 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1009 {
1010         struct cfiscsi_data_wait *cdw, *tmpcdw;
1011         union ctl_io *io;
1012         int error;
1013
1014 #ifdef notyet
1015         io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
1016         if (io == NULL) {
1017                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io");
1018                 return;
1019         }
1020         ctl_zero_io(io);
1021         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = NULL;
1022         io->io_hdr.io_type = CTL_IO_TASK;
1023         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1024         io->io_hdr.nexus.targ_port = cs->cs_target->ct_softc->fe.targ_port;
1025         io->io_hdr.nexus.targ_target.id = 0;
1026         io->io_hdr.nexus.targ_lun = lun;
1027         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1028         io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
1029         error = ctl_queue(io);
1030         if (error != CTL_RETVAL_COMPLETE) {
1031                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1032                 ctl_free_io(io);
1033         }
1034 #else
1035         /*
1036          * CTL doesn't currently support CTL_TASK_ABORT_TASK_SET, so instead
1037          * just iterate over tasks that are waiting for something - data - and
1038          * terminate those.
1039          */
1040         CFISCSI_SESSION_LOCK(cs);
1041         TAILQ_FOREACH_SAFE(cdw,
1042             &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
1043                 io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
1044                 if (io == NULL) {
1045                         CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io");
1046                         return;
1047                 }
1048                 ctl_zero_io(io);
1049                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = NULL;
1050                 io->io_hdr.io_type = CTL_IO_TASK;
1051                 io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1052                 io->io_hdr.nexus.targ_port =
1053                     cs->cs_target->ct_softc->fe.targ_port;
1054                 io->io_hdr.nexus.targ_target.id = 0;
1055                 //io->io_hdr.nexus.targ_lun = lun; /* Not needed? */
1056                 io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1057                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
1058                 io->taskio.tag_num = cdw->cdw_initiator_task_tag;
1059                 error = ctl_queue(io);
1060                 if (error != CTL_RETVAL_COMPLETE) {
1061                         CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1062                         ctl_free_io(io);
1063                         return;
1064                 }
1065 #if 0
1066                 CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task tag "
1067                     "0x%x", cdw->cdw_initiator_task_tag);
1068 #endif
1069                 cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1070                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1071                 uma_zfree(cfiscsi_data_wait_zone, cdw);
1072         }
1073         CFISCSI_SESSION_UNLOCK(cs);
1074 #endif
1075 }
1076
1077 static void
1078 cfiscsi_maintenance_thread(void *arg)
1079 {
1080         struct cfiscsi_session *cs;
1081
1082         cs = arg;
1083
1084         for (;;) {
1085                 CFISCSI_SESSION_LOCK(cs);
1086                 if (cs->cs_terminating == false)
1087                         cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1088                 CFISCSI_SESSION_UNLOCK(cs);
1089
1090                 if (cs->cs_terminating) {
1091                         cfiscsi_session_terminate_tasks(cs);
1092                         callout_drain(&cs->cs_callout);
1093
1094                         icl_conn_shutdown(cs->cs_conn);
1095                         icl_conn_close(cs->cs_conn);
1096
1097                         cs->cs_terminating++;
1098
1099                         /*
1100                          * XXX: We used to wait up to 30 seconds to deliver queued PDUs
1101                          *      to the initiator.  We also tried hard to deliver SCSI Responses
1102                          *      for the aborted PDUs.  We don't do that anymore.  We might need
1103                          *      to revisit that.
1104                          */
1105
1106                         cfiscsi_session_delete(cs);
1107                         kthread_exit();
1108                         return;
1109                 }
1110                 CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1111         }
1112 }
1113
1114 static void
1115 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1116 {
1117
1118         if (cs->cs_terminating != 0)
1119                 return;
1120         cs->cs_terminating = 1;
1121         cv_signal(&cs->cs_maintenance_cv);
1122 }
1123
1124 static int
1125 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1126 {
1127         int error, i;
1128         struct cfiscsi_softc *softc;
1129
1130         KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1131
1132         softc = &cfiscsi_softc;
1133
1134         mtx_lock(&softc->lock);
1135         for (i = 0; i < softc->max_initiators; i++) {
1136                 if (softc->ctl_initids[i] == 0)
1137                         break;
1138         }
1139         if (i == softc->max_initiators) {
1140                 CFISCSI_SESSION_WARN(cs, "too many concurrent sessions (%d)",
1141                     softc->max_initiators);
1142                 mtx_unlock(&softc->lock);
1143                 return (1);
1144         }
1145         softc->ctl_initids[i] = 1;
1146         mtx_unlock(&softc->lock);
1147
1148 #if 0
1149         CFISCSI_SESSION_DEBUG(cs, "adding initiator id %d, max %d",
1150             i, softc->max_initiators);
1151 #endif
1152         cs->cs_ctl_initid = i;
1153         error = ctl_add_initiator(0x0, softc->fe.targ_port, cs->cs_ctl_initid);
1154         if (error != 0) {
1155                 CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d", error);
1156                 mtx_lock(&softc->lock);
1157                 softc->ctl_initids[cs->cs_ctl_initid] = 0;
1158                 mtx_unlock(&softc->lock);
1159                 cs->cs_ctl_initid = -1;
1160                 return (1);
1161         }
1162
1163         return (0);
1164 }
1165
1166 static void
1167 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1168 {
1169         int error;
1170         struct cfiscsi_softc *softc;
1171
1172         if (cs->cs_ctl_initid == -1)
1173                 return;
1174
1175         softc = &cfiscsi_softc;
1176
1177         error = ctl_remove_initiator(softc->fe.targ_port, cs->cs_ctl_initid);
1178         if (error != 0) {
1179                 CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1180                     error);
1181         }
1182         mtx_lock(&softc->lock);
1183         softc->ctl_initids[cs->cs_ctl_initid] = 0;
1184         mtx_unlock(&softc->lock);
1185         cs->cs_ctl_initid = -1;
1186 }
1187
1188 static struct cfiscsi_session *
1189 cfiscsi_session_new(struct cfiscsi_softc *softc)
1190 {
1191         struct cfiscsi_session *cs;
1192         int error;
1193
1194         cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1195         if (cs == NULL) {
1196                 CFISCSI_WARN("malloc failed");
1197                 return (NULL);
1198         }
1199         cs->cs_ctl_initid = -1;
1200
1201         refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1202         TAILQ_INIT(&cs->cs_waiting_for_data_out);
1203         mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1204         cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1205 #ifdef ICL_KERNEL_PROXY
1206         cv_init(&cs->cs_login_cv, "cfiscsi_login");
1207 #endif
1208
1209         cs->cs_conn = icl_conn_new();
1210         cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1211         cs->cs_conn->ic_error = cfiscsi_error_callback;
1212         cs->cs_conn->ic_prv0 = cs;
1213
1214         error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1215         if (error != 0) {
1216                 CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1217                 free(cs, M_CFISCSI);
1218                 return (NULL);
1219         }
1220
1221         mtx_lock(&softc->lock);
1222         cs->cs_id = softc->last_session_id + 1;
1223         softc->last_session_id++;
1224         mtx_unlock(&softc->lock);
1225
1226         mtx_lock(&softc->lock);
1227         TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1228         mtx_unlock(&softc->lock);
1229
1230         /*
1231          * Start pinging the initiator.
1232          */
1233         callout_init(&cs->cs_callout, 1);
1234         callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1235
1236         return (cs);
1237 }
1238
1239 static void
1240 cfiscsi_session_delete(struct cfiscsi_session *cs)
1241 {
1242         struct cfiscsi_softc *softc;
1243
1244         softc = &cfiscsi_softc;
1245
1246         KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1247             ("destroying session with outstanding CTL pdus"));
1248         KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1249             ("destroying session with non-empty queue"));
1250
1251         cfiscsi_session_unregister_initiator(cs);
1252         if (cs->cs_target != NULL)
1253                 cfiscsi_target_release(cs->cs_target);
1254         icl_conn_close(cs->cs_conn);
1255         icl_conn_free(cs->cs_conn);
1256
1257         mtx_lock(&softc->lock);
1258         TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1259         mtx_unlock(&softc->lock);
1260
1261         free(cs, M_CFISCSI);
1262 }
1263
1264 int
1265 cfiscsi_init(void)
1266 {
1267         struct cfiscsi_softc *softc;
1268         struct ctl_frontend *fe;
1269         int retval;
1270
1271         softc = &cfiscsi_softc;
1272         retval = 0;
1273         bzero(softc, sizeof(*softc));
1274         mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1275
1276 #ifdef ICL_KERNEL_PROXY
1277         cv_init(&softc->accept_cv, "cfiscsi_accept");
1278 #endif
1279         TAILQ_INIT(&softc->sessions);
1280         TAILQ_INIT(&softc->targets);
1281
1282         fe = &softc->fe;
1283         fe->port_type = CTL_PORT_ISCSI;
1284         /* XXX KDM what should the real number be here? */
1285         fe->num_requested_ctl_io = 4096;
1286         snprintf(softc->port_name, sizeof(softc->port_name), "iscsi");
1287         fe->port_name = softc->port_name;
1288         fe->port_online = cfiscsi_online;
1289         fe->port_offline = cfiscsi_offline;
1290         fe->onoff_arg = softc;
1291         fe->targ_enable = cfiscsi_targ_enable;
1292         fe->targ_disable = cfiscsi_targ_disable;
1293         fe->lun_enable = cfiscsi_lun_enable;
1294         fe->lun_disable = cfiscsi_lun_disable;
1295         fe->targ_lun_arg = softc;
1296         fe->ioctl = cfiscsi_ioctl;
1297         fe->devid = cfiscsi_devid;
1298         fe->fe_datamove = cfiscsi_datamove;
1299         fe->fe_done = cfiscsi_done;
1300
1301         /* XXX KDM what should we report here? */
1302         /* XXX These should probably be fetched from CTL. */
1303         fe->max_targets = 1;
1304         fe->max_target_id = 15;
1305
1306         retval = ctl_frontend_register(fe, /*master_SC*/ 1);
1307         if (retval != 0) {
1308                 CFISCSI_WARN("ctl_frontend_register() failed with error %d",
1309                     retval);
1310                 retval = 1;
1311                 goto bailout;
1312         }
1313
1314         softc->max_initiators = fe->max_initiators;
1315
1316         cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1317             sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1318             UMA_ALIGN_PTR, 0);
1319
1320         return (0);
1321
1322 bailout:
1323         return (retval);
1324 }
1325
1326 static int
1327 cfiscsi_module_event_handler(module_t mod, int what, void *arg)
1328 {
1329
1330         switch (what) {
1331         case MOD_LOAD:
1332                 return (cfiscsi_init());
1333         case MOD_UNLOAD:
1334                 return (EBUSY);
1335         default:
1336                 return (EOPNOTSUPP);
1337         }
1338 }
1339
1340 #ifdef ICL_KERNEL_PROXY
1341 static void
1342 cfiscsi_accept(struct socket *so)
1343 {
1344         struct cfiscsi_session *cs;
1345
1346         cs = cfiscsi_session_new(&cfiscsi_softc);
1347         if (cs == NULL) {
1348                 CFISCSI_WARN("failed to create session");
1349                 return;
1350         }
1351
1352         icl_conn_handoff_sock(cs->cs_conn, so);
1353         cs->cs_waiting_for_ctld = true;
1354         cv_signal(&cfiscsi_softc.accept_cv);
1355 }
1356 #endif
1357
1358 static void
1359 cfiscsi_online(void *arg)
1360 {
1361         struct cfiscsi_softc *softc;
1362
1363         softc = (struct cfiscsi_softc *)arg;
1364
1365         softc->online = 1;
1366 #ifdef ICL_KERNEL_PROXY
1367         if (softc->listener != NULL)
1368                 icl_listen_free(softc->listener);
1369         softc->listener = icl_listen_new(cfiscsi_accept);
1370 #endif
1371 }
1372
1373 static void
1374 cfiscsi_offline(void *arg)
1375 {
1376         struct cfiscsi_softc *softc;
1377         struct cfiscsi_session *cs;
1378
1379         softc = (struct cfiscsi_softc *)arg;
1380
1381         softc->online = 0;
1382
1383         mtx_lock(&softc->lock);
1384         TAILQ_FOREACH(cs, &softc->sessions, cs_next)
1385                 cfiscsi_session_terminate(cs);
1386         mtx_unlock(&softc->lock);
1387
1388 #ifdef ICL_KERNEL_PROXY
1389         icl_listen_free(softc->listener);
1390         softc->listener = NULL;
1391 #endif
1392 }
1393
1394 static int
1395 cfiscsi_targ_enable(void *arg, struct ctl_id targ_id)
1396 {
1397
1398         return (0);
1399 }
1400
1401 static int
1402 cfiscsi_targ_disable(void *arg, struct ctl_id targ_id)
1403 {
1404
1405         return (0);
1406 }
1407
1408 static void
1409 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1410 {
1411         struct cfiscsi_softc *softc;
1412         struct cfiscsi_session *cs;
1413         struct cfiscsi_target *ct;
1414         struct ctl_iscsi_handoff_params *cihp;
1415 #ifndef ICL_KERNEL_PROXY
1416         int error;
1417 #endif
1418
1419         cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1420         softc = &cfiscsi_softc;
1421
1422         CFISCSI_DEBUG("new connection from %s (%s) to %s",
1423             cihp->initiator_name, cihp->initiator_addr,
1424             cihp->target_name);
1425
1426         if (softc->online == 0) {
1427                 ci->status = CTL_ISCSI_ERROR;
1428                 snprintf(ci->error_str, sizeof(ci->error_str),
1429                     "%s: port offline", __func__);
1430                 return;
1431         }
1432
1433         ct = cfiscsi_target_find(softc, cihp->target_name);
1434         if (ct == NULL) {
1435                 ci->status = CTL_ISCSI_ERROR;
1436                 snprintf(ci->error_str, sizeof(ci->error_str),
1437                     "%s: target not found", __func__);
1438                 return;
1439         }
1440
1441 #ifdef ICL_KERNEL_PROXY
1442         mtx_lock(&cfiscsi_softc.lock);
1443         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1444                 if (cs->cs_id == cihp->socket)
1445                         break;
1446         }
1447         if (cs == NULL) {
1448                 mtx_unlock(&cfiscsi_softc.lock);
1449                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1450                 ci->status = CTL_ISCSI_ERROR;
1451                 return;
1452         }
1453         mtx_unlock(&cfiscsi_softc.lock);
1454 #else
1455         cs = cfiscsi_session_new(softc);
1456         if (cs == NULL) {
1457                 ci->status = CTL_ISCSI_ERROR;
1458                 snprintf(ci->error_str, sizeof(ci->error_str),
1459                     "%s: cfiscsi_session_new failed", __func__);
1460                 cfiscsi_target_release(ct);
1461                 return;
1462         }
1463 #endif
1464         cs->cs_target = ct;
1465
1466         /*
1467          * First PDU of Full Feature phase has the same CmdSN as the last
1468          * PDU from the Login Phase received from the initiator.  Thus,
1469          * the -1 below.
1470          */
1471         cs->cs_portal_group_tag = cihp->portal_group_tag;
1472         cs->cs_cmdsn = cihp->cmdsn;
1473         cs->cs_statsn = cihp->statsn;
1474         cs->cs_max_data_segment_length = cihp->max_recv_data_segment_length;
1475         cs->cs_max_burst_length = cihp->max_burst_length;
1476         cs->cs_immediate_data = !!cihp->immediate_data;
1477         if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1478                 cs->cs_conn->ic_header_crc32c = true;
1479         if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1480                 cs->cs_conn->ic_data_crc32c = true;
1481
1482         strlcpy(cs->cs_initiator_name,
1483             cihp->initiator_name, sizeof(cs->cs_initiator_name));
1484         strlcpy(cs->cs_initiator_addr,
1485             cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1486         strlcpy(cs->cs_initiator_alias,
1487             cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1488
1489 #ifdef ICL_KERNEL_PROXY
1490         cs->cs_login_phase = false;
1491 #else
1492         error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1493         if (error != 0) {
1494                 cfiscsi_session_delete(cs);
1495                 ci->status = CTL_ISCSI_ERROR;
1496                 snprintf(ci->error_str, sizeof(ci->error_str),
1497                     "%s: icl_conn_handoff failed with error %d",
1498                     __func__, error);
1499                 return;
1500         }
1501 #endif
1502
1503         /*
1504          * Register initiator with CTL.
1505          */
1506         cfiscsi_session_register_initiator(cs);
1507
1508 #ifdef ICL_KERNEL_PROXY
1509         /*
1510          * First PDU of the Full Feature phase has likely already arrived.
1511          * We have to pick it up and execute properly.
1512          */
1513         if (cs->cs_login_pdu != NULL) {
1514                 CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1515                 cfiscsi_pdu_handle(cs->cs_login_pdu);
1516                 cs->cs_login_pdu = NULL;
1517         }
1518 #endif
1519
1520         ci->status = CTL_ISCSI_OK;
1521 }
1522
1523 static void
1524 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1525 {
1526         struct ctl_iscsi_list_params *cilp;
1527         struct cfiscsi_session *cs;
1528         struct cfiscsi_softc *softc;
1529         struct sbuf *sb;
1530         int error;
1531
1532         cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1533         softc = &cfiscsi_softc;
1534
1535         sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1536         if (sb == NULL) {
1537                 ci->status = CTL_ISCSI_ERROR;
1538                 snprintf(ci->error_str, sizeof(ci->error_str),
1539                     "Unable to allocate %d bytes for iSCSI session list",
1540                     cilp->alloc_len);
1541                 return;
1542         }
1543
1544         sbuf_printf(sb, "<ctlislist>\n");
1545         mtx_lock(&softc->lock);
1546         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1547 #ifdef ICL_KERNEL_PROXY
1548                 if (cs->cs_target == NULL)
1549                         continue;
1550 #endif
1551                 error = sbuf_printf(sb, "<connection id=\"%d\">"
1552                     "<initiator>%s</initiator>"
1553                     "<initiator_addr>%s</initiator_addr>"
1554                     "<initiator_alias>%s</initiator_alias>"
1555                     "<target>%s</target>"
1556                     "<target_alias>%s</target_alias>"
1557                     "<header_digest>%s</header_digest>"
1558                     "<data_digest>%s</data_digest>"
1559                     "<max_data_segment_length>%zd</max_data_segment_length>"
1560                     "<immediate_data>%d</immediate_data>"
1561                     "<iser>%d</iser>"
1562                     "</connection>\n",
1563                     cs->cs_id,
1564                     cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1565                     cs->cs_target->ct_name, cs->cs_target->ct_alias,
1566                     cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1567                     cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1568                     cs->cs_max_data_segment_length,
1569                     cs->cs_immediate_data,
1570                     cs->cs_conn->ic_iser);
1571                 if (error != 0)
1572                         break;
1573         }
1574         mtx_unlock(&softc->lock);
1575         error = sbuf_printf(sb, "</ctlislist>\n");
1576         if (error != 0) {
1577                 sbuf_delete(sb);
1578                 ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1579                 snprintf(ci->error_str, sizeof(ci->error_str),
1580                     "Out of space, %d bytes is too small", cilp->alloc_len);
1581                 return;
1582         }
1583         sbuf_finish(sb);
1584
1585         error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1586         cilp->fill_len = sbuf_len(sb) + 1;
1587         ci->status = CTL_ISCSI_OK;
1588         sbuf_delete(sb);
1589 }
1590
1591 static void
1592 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1593 {
1594         struct icl_pdu *response;
1595         struct iscsi_bhs_asynchronous_message *bhsam;
1596         struct ctl_iscsi_terminate_params *citp;
1597         struct cfiscsi_session *cs;
1598         struct cfiscsi_softc *softc;
1599         int found = 0;
1600
1601         citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1602         softc = &cfiscsi_softc;
1603
1604         mtx_lock(&softc->lock);
1605         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1606                 if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1607                     strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1608                     strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1609                         continue;
1610
1611                 response = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
1612                 if (response == NULL) {
1613                         /*
1614                          * Oh well.  Just terminate the connection.
1615                          */
1616                 } else {
1617                         bhsam = (struct iscsi_bhs_asynchronous_message *)
1618                             response->ip_bhs;
1619                         bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1620                         bhsam->bhsam_flags = 0x80;
1621                         bhsam->bhsam_0xffffffff = 0xffffffff;
1622                         bhsam->bhsam_async_event =
1623                             BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1624                         cfiscsi_pdu_queue(response);
1625                 }
1626                 cfiscsi_session_terminate(cs);
1627                 found++;
1628         }
1629         mtx_unlock(&softc->lock);
1630
1631         if (found == 0) {
1632                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1633                 snprintf(ci->error_str, sizeof(ci->error_str),
1634                     "No matching connections found");
1635                 return;
1636         }
1637
1638         ci->status = CTL_ISCSI_OK;
1639 }
1640
1641 static void
1642 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1643 {
1644         struct icl_pdu *response;
1645         struct iscsi_bhs_asynchronous_message *bhsam;
1646         struct ctl_iscsi_logout_params *cilp;
1647         struct cfiscsi_session *cs;
1648         struct cfiscsi_softc *softc;
1649         int found = 0;
1650
1651         cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1652         softc = &cfiscsi_softc;
1653
1654         mtx_lock(&softc->lock);
1655         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1656                 if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1657                     strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1658                     strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1659                         continue;
1660
1661                 response = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
1662                 if (response == NULL) {
1663                         ci->status = CTL_ISCSI_ERROR;
1664                         snprintf(ci->error_str, sizeof(ci->error_str),
1665                             "Unable to allocate memory");
1666                         mtx_unlock(&softc->lock);
1667                         return;
1668                 }
1669                 bhsam =
1670                     (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1671                 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1672                 bhsam->bhsam_flags = 0x80;
1673                 bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1674                 bhsam->bhsam_parameter3 = htons(10);
1675                 cfiscsi_pdu_queue(response);
1676                 found++;
1677         }
1678         mtx_unlock(&softc->lock);
1679
1680         if (found == 0) {
1681                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1682                 snprintf(ci->error_str, sizeof(ci->error_str),
1683                     "No matching connections found");
1684                 return;
1685         }
1686
1687         ci->status = CTL_ISCSI_OK;
1688 }
1689
1690 #ifdef ICL_KERNEL_PROXY
1691 static void
1692 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1693 {
1694         struct ctl_iscsi_listen_params *cilp;
1695         struct sockaddr *sa;
1696         int error;
1697
1698         cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1699
1700         if (cfiscsi_softc.listener == NULL) {
1701                 CFISCSI_DEBUG("no listener");
1702                 snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1703                 ci->status = CTL_ISCSI_ERROR;
1704                 return;
1705         }
1706
1707         error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1708         if (error != 0) {
1709                 CFISCSI_DEBUG("getsockaddr, error %d", error);
1710                 snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1711                 ci->status = CTL_ISCSI_ERROR;
1712                 return;
1713         }
1714
1715         error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1716             cilp->socktype, cilp->protocol, sa);
1717         if (error != 0) {
1718                 free(sa, M_SONAME);
1719                 CFISCSI_DEBUG("icl_listen_add, error %d", error);
1720                 snprintf(ci->error_str, sizeof(ci->error_str),
1721                     "icl_listen_add failed, error %d", error);
1722                 ci->status = CTL_ISCSI_ERROR;
1723                 return;
1724         }
1725
1726         ci->status = CTL_ISCSI_OK;
1727 }
1728
1729 static void
1730 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1731 {
1732         struct ctl_iscsi_accept_params *ciap;
1733         struct cfiscsi_session *cs;
1734         int error;
1735
1736         ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1737
1738         mtx_lock(&cfiscsi_softc.lock);
1739         for (;;) {
1740                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1741                         if (cs->cs_waiting_for_ctld)
1742                                 break;
1743                 }
1744                 if (cs != NULL)
1745                         break;
1746                 error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1747                 if (error != 0) {
1748                         mtx_unlock(&cfiscsi_softc.lock);
1749                         snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1750                         ci->status = CTL_ISCSI_ERROR;
1751                         return;
1752                 }
1753         }
1754         mtx_unlock(&cfiscsi_softc.lock);
1755
1756         cs->cs_waiting_for_ctld = false;
1757         cs->cs_login_phase = true;
1758
1759         ciap->connection_id = cs->cs_id;
1760         ci->status = CTL_ISCSI_OK;
1761 }
1762
1763 static void
1764 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1765 {
1766         struct ctl_iscsi_send_params *cisp;
1767         struct cfiscsi_session *cs;
1768         struct icl_pdu *ip;
1769         size_t datalen;
1770         void *data;
1771         int error;
1772
1773         cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1774
1775         mtx_lock(&cfiscsi_softc.lock);
1776         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1777                 if (cs->cs_id == cisp->connection_id)
1778                         break;
1779         }
1780         if (cs == NULL) {
1781                 mtx_unlock(&cfiscsi_softc.lock);
1782                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1783                 ci->status = CTL_ISCSI_ERROR;
1784                 return;
1785         }
1786         mtx_unlock(&cfiscsi_softc.lock);
1787
1788 #if 0
1789         if (cs->cs_login_phase == false)
1790                 return (EBUSY);
1791 #endif
1792
1793         if (cs->cs_terminating) {
1794                 snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1795                 ci->status = CTL_ISCSI_ERROR;
1796                 return;
1797         }
1798
1799         datalen = cisp->data_segment_len;
1800         /*
1801          * XXX
1802          */
1803         //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
1804         if (datalen > 65535) {
1805                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1806                 ci->status = CTL_ISCSI_ERROR;
1807                 return;
1808         }
1809         if (datalen > 0) {
1810                 data = malloc(datalen, M_CFISCSI, M_WAITOK);
1811                 error = copyin(cisp->data_segment, data, datalen);
1812                 if (error != 0) {
1813                         free(data, M_CFISCSI);
1814                         snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
1815                         ci->status = CTL_ISCSI_ERROR;
1816                         return;
1817                 }
1818         }
1819
1820         ip = icl_pdu_new_bhs(cs->cs_conn, M_WAITOK);
1821         memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
1822         if (datalen > 0) {
1823                 icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1824                 free(data, M_CFISCSI);
1825         }
1826         icl_pdu_queue(ip);
1827         ci->status = CTL_ISCSI_OK;
1828 }
1829
1830 static void
1831 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
1832 {
1833         struct ctl_iscsi_receive_params *cirp;
1834         struct cfiscsi_session *cs;
1835         struct icl_pdu *ip;
1836         void *data;
1837
1838         cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
1839
1840         mtx_lock(&cfiscsi_softc.lock);
1841         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1842                 if (cs->cs_id == cirp->connection_id)
1843                         break;
1844         }
1845         if (cs == NULL) {
1846                 mtx_unlock(&cfiscsi_softc.lock);
1847                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1848                 ci->status = CTL_ISCSI_ERROR;
1849                 return;
1850         }
1851         mtx_unlock(&cfiscsi_softc.lock);
1852
1853 #if 0
1854         if (is->is_login_phase == false)
1855                 return (EBUSY);
1856 #endif
1857
1858         CFISCSI_SESSION_LOCK(cs);
1859         while (cs->cs_login_pdu == NULL &&
1860             cs->cs_terminating == false)
1861                 cv_wait(&cs->cs_login_cv, &cs->cs_lock);
1862         if (cs->cs_terminating) {
1863                 CFISCSI_SESSION_UNLOCK(cs);
1864                 snprintf(ci->error_str, sizeof(ci->error_str), "connection terminating");
1865                 ci->status = CTL_ISCSI_ERROR;
1866                 return;
1867         }
1868         ip = cs->cs_login_pdu;
1869         cs->cs_login_pdu = NULL;
1870         CFISCSI_SESSION_UNLOCK(cs);
1871
1872         if (ip->ip_data_len > cirp->data_segment_len) {
1873                 icl_pdu_free(ip);
1874                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1875                 ci->status = CTL_ISCSI_ERROR;
1876                 return;
1877         }
1878
1879         copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
1880         if (ip->ip_data_len > 0) {
1881                 data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
1882                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1883                 copyout(data, cirp->data_segment, ip->ip_data_len);
1884                 free(data, M_CFISCSI);
1885         }
1886
1887         icl_pdu_free(ip);
1888         ci->status = CTL_ISCSI_OK;
1889 }
1890
1891 static void
1892 cfiscsi_ioctl_close(struct ctl_iscsi *ci)
1893 {
1894         /*
1895          * XXX
1896          */
1897 }
1898 #endif /* !ICL_KERNEL_PROXY */
1899
1900 static int
1901 cfiscsi_ioctl(struct cdev *dev,
1902     u_long cmd, caddr_t addr, int flag, struct thread *td)
1903 {
1904         struct ctl_iscsi *ci;
1905
1906         if (cmd != CTL_ISCSI)
1907                 return (ENOTTY);
1908
1909         ci = (struct ctl_iscsi *)addr;
1910         switch (ci->type) {
1911         case CTL_ISCSI_HANDOFF:
1912                 cfiscsi_ioctl_handoff(ci);
1913                 break;
1914         case CTL_ISCSI_LIST:
1915                 cfiscsi_ioctl_list(ci);
1916                 break;
1917         case CTL_ISCSI_TERMINATE:
1918                 cfiscsi_ioctl_terminate(ci);
1919                 break;
1920         case CTL_ISCSI_LOGOUT:
1921                 cfiscsi_ioctl_logout(ci);
1922                 break;
1923 #ifdef ICL_KERNEL_PROXY
1924         case CTL_ISCSI_LISTEN:
1925                 cfiscsi_ioctl_listen(ci);
1926                 break;
1927         case CTL_ISCSI_ACCEPT:
1928                 cfiscsi_ioctl_accept(ci);
1929                 break;
1930         case CTL_ISCSI_SEND:
1931                 cfiscsi_ioctl_send(ci);
1932                 break;
1933         case CTL_ISCSI_RECEIVE:
1934                 cfiscsi_ioctl_receive(ci);
1935                 break;
1936         case CTL_ISCSI_CLOSE:
1937                 cfiscsi_ioctl_close(ci);
1938                 break;
1939 #endif /* ICL_KERNEL_PROXY */
1940         default:
1941                 ci->status = CTL_ISCSI_ERROR;
1942                 snprintf(ci->error_str, sizeof(ci->error_str),
1943                     "%s: invalid iSCSI request type %d", __func__, ci->type);
1944                 break;
1945         }
1946
1947         return (0);
1948 }
1949
1950 static int
1951 cfiscsi_devid(struct ctl_scsiio *ctsio, int alloc_len)
1952 {
1953         struct cfiscsi_session *cs;
1954         struct scsi_vpd_device_id *devid_ptr;
1955         struct scsi_vpd_id_descriptor *desc, *desc1;
1956         struct scsi_vpd_id_descriptor *desc2, *desc3; /* for types 4h and 5h */
1957         struct scsi_vpd_id_t10 *t10id;
1958         struct ctl_lun *lun;
1959         const struct icl_pdu *request;
1960         size_t devid_len, wwpn_len;
1961
1962         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
1963         request = ctsio->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1964         cs = PDU_SESSION(request);
1965
1966         wwpn_len = strlen(cs->cs_target->ct_name);
1967         wwpn_len += strlen(",t,0x01");
1968         wwpn_len += 1; /* '\0' */
1969         if ((wwpn_len % 4) != 0)
1970                 wwpn_len += (4 - (wwpn_len % 4));
1971
1972         devid_len = sizeof(struct scsi_vpd_device_id) +
1973                 sizeof(struct scsi_vpd_id_descriptor) +
1974                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN +
1975                 sizeof(struct scsi_vpd_id_descriptor) + wwpn_len +
1976                 sizeof(struct scsi_vpd_id_descriptor) +
1977                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
1978                 sizeof(struct scsi_vpd_id_descriptor) +
1979                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
1980
1981         ctsio->kern_data_ptr = malloc(devid_len, M_CTL, M_WAITOK | M_ZERO);
1982         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
1983         ctsio->kern_sg_entries = 0;
1984
1985         if (devid_len < alloc_len) {
1986                 ctsio->residual = alloc_len - devid_len;
1987                 ctsio->kern_data_len = devid_len;
1988                 ctsio->kern_total_len = devid_len;
1989         } else {
1990                 ctsio->residual = 0;
1991                 ctsio->kern_data_len = alloc_len;
1992                 ctsio->kern_total_len = alloc_len;
1993         }
1994         ctsio->kern_data_resid = 0;
1995         ctsio->kern_rel_offset = 0;
1996         ctsio->kern_sg_entries = 0;
1997
1998         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
1999         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
2000         desc1 = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
2001             sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN);
2002         desc2 = (struct scsi_vpd_id_descriptor *)(&desc1->identifier[0] +
2003             wwpn_len);
2004         desc3 = (struct scsi_vpd_id_descriptor *)(&desc2->identifier[0] +
2005             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
2006
2007         if (lun != NULL)
2008                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
2009                     lun->be_lun->lun_type;
2010         else
2011                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
2012
2013         devid_ptr->page_code = SVPD_DEVICE_ID;
2014
2015         scsi_ulto2b(devid_len - 4, devid_ptr->length);
2016
2017         /*
2018          * We're using a LUN association here.  i.e., this device ID is a
2019          * per-LUN identifier.
2020          */
2021         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_ASCII;
2022         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
2023         desc->length = sizeof(*t10id) + CTL_DEVID_LEN;
2024         strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
2025
2026         /*
2027          * If we've actually got a backend, copy the device id from the
2028          * per-LUN data.  Otherwise, set it to all spaces.
2029          */
2030         if (lun != NULL) {
2031                 /*
2032                  * Copy the backend's LUN ID.
2033                  */
2034                 strncpy((char *)t10id->vendor_spec_id,
2035                     (char *)lun->be_lun->device_id, CTL_DEVID_LEN);
2036         } else {
2037                 /*
2038                  * No backend, set this to spaces.
2039                  */
2040                 memset(t10id->vendor_spec_id, 0x20, CTL_DEVID_LEN);
2041         }
2042
2043         /*
2044          * desc1 is for the WWPN which is a port asscociation.
2045          */
2046         desc1->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2047         desc1->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2048             SVPD_ID_TYPE_SCSI_NAME;
2049         desc1->length = wwpn_len;
2050         snprintf(desc1->identifier, wwpn_len, "%s,t,0x%x",
2051             cs->cs_target->ct_name, cs->cs_portal_group_tag);
2052
2053         /*
2054          * desc2 is for the Relative Target Port(type 4h) identifier
2055          */
2056         desc2->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_BINARY;
2057         desc2->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2058             SVPD_ID_TYPE_RELTARG;
2059         desc2->length = 4;
2060         desc2->identifier[3] = 1;
2061
2062         /*
2063          * desc3 is for the Target Port Group(type 5h) identifier
2064          */
2065         desc3->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_BINARY;
2066         desc3->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2067             SVPD_ID_TYPE_TPORTGRP;
2068         desc3->length = 4;
2069         desc3->identifier[3] = 1;
2070
2071         ctsio->scsi_status = SCSI_STATUS_OK;
2072
2073         ctsio->be_move_done = ctl_config_move_done;
2074         ctl_datamove((union ctl_io *)ctsio);
2075
2076         return (CTL_RETVAL_COMPLETE);
2077 }
2078
2079 static void
2080 cfiscsi_target_hold(struct cfiscsi_target *ct)
2081 {
2082
2083         refcount_acquire(&ct->ct_refcount);
2084 }
2085
2086 static void
2087 cfiscsi_target_release(struct cfiscsi_target *ct)
2088 {
2089         int old;
2090         struct cfiscsi_softc *softc;
2091
2092         softc = ct->ct_softc;
2093
2094         old = ct->ct_refcount;
2095         if (old > 1 && atomic_cmpset_int(&ct->ct_refcount, old, old - 1))
2096                 return;
2097
2098         mtx_lock(&softc->lock);
2099         if (refcount_release(&ct->ct_refcount)) {
2100                 TAILQ_REMOVE(&softc->targets, ct, ct_next);
2101                 mtx_unlock(&softc->lock);
2102                 free(ct, M_CFISCSI);
2103
2104                 return;
2105         }
2106         mtx_unlock(&softc->lock);
2107 }
2108
2109 static struct cfiscsi_target *
2110 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name)
2111 {
2112         struct cfiscsi_target *ct;
2113
2114         mtx_lock(&softc->lock);
2115         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2116                 if (strcmp(name, ct->ct_name) != 0)
2117                         continue;
2118                 cfiscsi_target_hold(ct);
2119                 mtx_unlock(&softc->lock);
2120                 return (ct);
2121         }
2122         mtx_unlock(&softc->lock);
2123
2124         return (NULL);
2125 }
2126
2127 static struct cfiscsi_target *
2128 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2129     const char *alias)
2130 {
2131         struct cfiscsi_target *ct, *newct;
2132         int i;
2133
2134         if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2135                 return (NULL);
2136
2137         newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2138
2139         mtx_lock(&softc->lock);
2140         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2141                 if (strcmp(name, ct->ct_name) != 0)
2142                         continue;
2143                 cfiscsi_target_hold(ct);
2144                 mtx_unlock(&softc->lock);
2145                 free(newct, M_CFISCSI);
2146                 return (ct);
2147         }
2148
2149         for (i = 0; i < CTL_MAX_LUNS; i++)
2150                 newct->ct_luns[i] = -1;
2151
2152         strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2153         if (alias != NULL)
2154                 strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2155         refcount_init(&newct->ct_refcount, 1);
2156         newct->ct_softc = softc;
2157         TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2158         mtx_unlock(&softc->lock);
2159
2160         return (newct);
2161 }
2162
2163 /*
2164  * Takes LUN from the target space and returns LUN from the CTL space.
2165  */
2166 static uint32_t
2167 cfiscsi_map_lun(void *arg, uint32_t lun)
2168 {
2169         struct cfiscsi_session *cs;
2170
2171         cs = arg;
2172
2173         if (lun >= CTL_MAX_LUNS) {
2174                 CFISCSI_DEBUG("requested lun number %d is higher "
2175                     "than maximum %d", lun, CTL_MAX_LUNS - 1);
2176                 return (0xffffffff);
2177         }
2178
2179         if (cs->cs_target->ct_luns[lun] < 0)
2180                 return (0xffffffff);
2181
2182         return (cs->cs_target->ct_luns[lun]);
2183 }
2184
2185 static int
2186 cfiscsi_target_set_lun(struct cfiscsi_target *ct,
2187     unsigned long lun_id, unsigned long ctl_lun_id)
2188 {
2189
2190         if (lun_id >= CTL_MAX_LUNS) {
2191                 CFISCSI_WARN("requested lun number %ld is higher "
2192                     "than maximum %d", lun_id, CTL_MAX_LUNS - 1);
2193                 return (-1);
2194         }
2195
2196         if (ct->ct_luns[lun_id] >= 0) {
2197                 /*
2198                  * CTL calls cfiscsi_lun_enable() twice for each LUN - once
2199                  * when the LUN is created, and a second time just before
2200                  * the port is brought online; don't emit warnings
2201                  * for that case.
2202                  */
2203                 if (ct->ct_luns[lun_id] == ctl_lun_id)
2204                         return (0);
2205                 CFISCSI_WARN("lun %ld already allocated", lun_id);
2206                 return (-1);
2207         }
2208
2209 #if 0
2210         CFISCSI_DEBUG("adding mapping for lun %ld, target %s "
2211             "to ctl lun %ld", lun_id, ct->ct_name, ctl_lun_id);
2212 #endif
2213
2214         ct->ct_luns[lun_id] = ctl_lun_id;
2215         cfiscsi_target_hold(ct);
2216
2217         return (0);
2218 }
2219
2220 static int
2221 cfiscsi_target_unset_lun(struct cfiscsi_target *ct, unsigned long lun_id)
2222 {
2223
2224         if (ct->ct_luns[lun_id] < 0) {
2225                 CFISCSI_WARN("lun %ld not allocated", lun_id);
2226                 return (-1);
2227         }
2228
2229         ct->ct_luns[lun_id] = -1;
2230         cfiscsi_target_release(ct);
2231
2232         return (0);
2233 }
2234
2235 static int
2236 cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id)
2237 {
2238         struct cfiscsi_softc *softc;
2239         struct cfiscsi_target *ct;
2240         struct ctl_be_lun_option *opt;
2241         const char *target = NULL, *target_alias = NULL;
2242         const char *lun = NULL;
2243         unsigned long tmp;
2244
2245         softc = (struct cfiscsi_softc *)arg;
2246
2247         STAILQ_FOREACH(opt,
2248             &control_softc->ctl_luns[lun_id]->be_lun->options, links) {
2249                 if (strcmp(opt->name, "cfiscsi_target") == 0)
2250                         target = opt->value;
2251                 else if (strcmp(opt->name, "cfiscsi_target_alias") == 0)
2252                         target_alias = opt->value;
2253                 else if (strcmp(opt->name, "cfiscsi_lun") == 0)
2254                         lun = opt->value;
2255         }
2256
2257         if (target == NULL && lun == NULL)
2258                 return (0);
2259
2260         if (target == NULL || lun == NULL) {
2261                 CFISCSI_WARN("lun added with cfiscsi_target, but without "
2262                     "cfiscsi_lun, or the other way around; ignoring");
2263                 return (0);
2264         }
2265
2266         ct = cfiscsi_target_find_or_create(softc, target, target_alias);
2267         if (ct == NULL) {
2268                 CFISCSI_WARN("failed to create target \"%s\"", target);
2269                 return (0);
2270         }
2271
2272         tmp = strtoul(lun, NULL, 10);
2273         cfiscsi_target_set_lun(ct, tmp, lun_id);
2274         return (0);
2275 }
2276
2277 static int
2278 cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
2279 {
2280         struct cfiscsi_softc *softc;
2281         struct cfiscsi_target *ct;
2282         int i;
2283
2284         softc = (struct cfiscsi_softc *)arg;
2285
2286         mtx_lock(&softc->lock);
2287         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2288                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2289                         if (ct->ct_luns[i] < 0)
2290                                 continue;
2291                         if (ct->ct_luns[i] != lun_id)
2292                                 continue;
2293                         cfiscsi_target_unset_lun(ct, i);
2294                         break;
2295                 }
2296         }
2297         mtx_unlock(&softc->lock);
2298         return (0);
2299 }
2300
2301 static void
2302 cfiscsi_datamove_in(union ctl_io *io)
2303 {
2304         struct cfiscsi_session *cs;
2305         struct icl_pdu *request, *response;
2306         const struct iscsi_bhs_scsi_command *bhssc;
2307         struct iscsi_bhs_data_in *bhsdi;
2308         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2309         size_t copy_len, len, off;
2310         const char *addr;
2311         int ctl_sg_count, error, i;
2312
2313         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2314         cs = PDU_SESSION(request);
2315
2316         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2317         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2318             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2319             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2320
2321         if (io->scsiio.kern_sg_entries > 0) {
2322                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2323                 ctl_sg_count = io->scsiio.kern_sg_entries;
2324         } else {
2325                 ctl_sglist = &ctl_sg_entry;
2326                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2327                 ctl_sglist->len = io->scsiio.kern_data_len;
2328                 ctl_sg_count = 1;
2329         }
2330
2331         /*
2332          * We need to record it so that we can properly report
2333          * underflow/underflow.
2334          */
2335         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2336
2337         /*
2338          * This is the offset within the current SCSI command;
2339          * i.e. for the first call of datamove(), it will be 0,
2340          * and for subsequent ones it will be the sum of lengths
2341          * of previous ones.
2342          */
2343         off = htonl(io->scsiio.kern_rel_offset);
2344
2345         i = 0;
2346         addr = NULL;
2347         len = 0;
2348         response = NULL;
2349         bhsdi = NULL;
2350         for (;;) {
2351                 KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2352                 if (response == NULL) {
2353                         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2354                         if (response == NULL) {
2355                                 CFISCSI_SESSION_WARN(cs, "failed to "
2356                                     "allocate memory; dropping connection");
2357                                 ctl_set_busy(&io->scsiio);
2358                                 io->scsiio.be_move_done(io);
2359                                 cfiscsi_session_terminate(cs);
2360                                 return;
2361                         }
2362                         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2363                         bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2364                         bhsdi->bhsdi_initiator_task_tag =
2365                             bhssc->bhssc_initiator_task_tag;
2366                         bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2367                         PDU_EXPDATASN(request)++;
2368                         bhsdi->bhsdi_buffer_offset = htonl(off);
2369                 }
2370
2371                 if (len == 0) {
2372                         addr = ctl_sglist[i].addr;
2373                         len = ctl_sglist[i].len;
2374                         KASSERT(len > 0, ("len <= 0"));
2375                 }
2376
2377                 copy_len = len;
2378                 if (response->ip_data_len + copy_len >
2379                     cs->cs_max_data_segment_length)
2380                         copy_len = cs->cs_max_data_segment_length -
2381                             response->ip_data_len;
2382                 KASSERT(copy_len <= len, ("copy_len > len"));
2383                 error = icl_pdu_append_data(response, addr, copy_len, M_NOWAIT);
2384                 if (error != 0) {
2385                         CFISCSI_SESSION_WARN(cs, "failed to "
2386                             "allocate memory; dropping connection");
2387                         icl_pdu_free(response);
2388                         ctl_set_busy(&io->scsiio);
2389                         io->scsiio.be_move_done(io);
2390                         cfiscsi_session_terminate(cs);
2391                         return;
2392                 }
2393                 addr += copy_len;
2394                 len -= copy_len;
2395                 off += copy_len;
2396                 io->scsiio.ext_data_filled += copy_len;
2397
2398                 if (len == 0) {
2399                         /*
2400                          * End of scatter-gather segment;
2401                          * proceed to the next one...
2402                          */
2403                         if (i == ctl_sg_count - 1) {
2404                                 /*
2405                                  * ... unless this was the last one.
2406                                  */
2407                                 break;
2408                         }
2409                         i++;
2410                 }
2411
2412                 if (response->ip_data_len == cs->cs_max_data_segment_length) {
2413                         /*
2414                          * Can't stuff more data into the current PDU;
2415                          * queue it.  Note that's not enough to check
2416                          * for kern_data_resid == 0 instead; there
2417                          * may be several Data-In PDUs for the final
2418                          * call to cfiscsi_datamove(), and we want
2419                          * to set the F flag only on the last of them.
2420                          */
2421                         if (off == io->scsiio.kern_total_len)
2422                                 bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2423                         KASSERT(response->ip_data_len > 0,
2424                             ("sending empty Data-In"));
2425                         cfiscsi_pdu_queue(response);
2426                         response = NULL;
2427                         bhsdi = NULL;
2428                 }
2429         }
2430         KASSERT(i == ctl_sg_count - 1, ("missed SG segment"));
2431         KASSERT(len == 0, ("missed data from SG segment"));
2432         if (response != NULL) {
2433                 if (off == io->scsiio.kern_total_len) {
2434                         bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2435 #if 0
2436                 } else {
2437                         CFISCSI_SESSION_DEBUG(cs, "not setting the F flag; "
2438                             "have %zd, need %zd", off,
2439                             (size_t)io->scsiio.kern_total_len);
2440 #endif
2441                 }
2442                 KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2443                 cfiscsi_pdu_queue(response);
2444         }
2445
2446         io->scsiio.be_move_done(io);
2447 }
2448
2449 static void
2450 cfiscsi_datamove_out(union ctl_io *io)
2451 {
2452         struct cfiscsi_session *cs;
2453         struct icl_pdu *request, *response;
2454         const struct iscsi_bhs_scsi_command *bhssc;
2455         struct iscsi_bhs_r2t *bhsr2t;
2456         struct cfiscsi_data_wait *cdw;
2457         uint32_t target_transfer_tag;
2458         bool done;
2459
2460         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2461         cs = PDU_SESSION(request);
2462
2463         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2464         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2465             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2466             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2467
2468         /*
2469          * We need to record it so that we can properly report
2470          * underflow/underflow.
2471          */
2472         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2473
2474         CFISCSI_SESSION_LOCK(cs);
2475         target_transfer_tag = cs->cs_target_transfer_tag;
2476         cs->cs_target_transfer_tag++;
2477         CFISCSI_SESSION_UNLOCK(cs);
2478
2479 #if 0
2480         CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2481             "task tag 0x%x, target transfer tag 0x%x",
2482             bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2483 #endif
2484         cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
2485         if (cdw == NULL) {
2486                 CFISCSI_SESSION_WARN(cs, "failed to "
2487                     "allocate memory; dropping connection");
2488                 ctl_set_busy(&io->scsiio);
2489                 io->scsiio.be_move_done(io);
2490                 cfiscsi_session_terminate(cs);
2491                 return;
2492         }
2493         cdw->cdw_ctl_io = io;
2494         cdw->cdw_target_transfer_tag = htonl(target_transfer_tag);
2495         cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2496
2497         if (cs->cs_immediate_data && icl_pdu_data_segment_length(request) > 0) {
2498                 done = cfiscsi_handle_data_segment(request, cdw);
2499                 if (done) {
2500                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2501                         io->scsiio.be_move_done(io);
2502                         return;
2503                 }
2504
2505 #if 0
2506                 if (io->scsiio.ext_data_filled != 0)
2507                         CFISCSI_SESSION_DEBUG(cs, "got %zd bytes of immediate data, need %zd",
2508                             io->scsiio.ext_data_filled, io->scsiio.kern_data_len);
2509 #endif
2510         }
2511
2512         CFISCSI_SESSION_LOCK(cs);
2513         TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2514         CFISCSI_SESSION_UNLOCK(cs);
2515
2516         /*
2517          * XXX: We should limit the number of outstanding R2T PDUs
2518          *      per task to MaxOutstandingR2T.
2519          */
2520         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2521         if (response == NULL) {
2522                 CFISCSI_SESSION_WARN(cs, "failed to "
2523                     "allocate memory; dropping connection");
2524                 ctl_set_busy(&io->scsiio);
2525                 io->scsiio.be_move_done(io);
2526                 cfiscsi_session_terminate(cs);
2527                 return;
2528         }
2529         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2530         bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2531         bhsr2t->bhsr2t_flags = 0x80;
2532         bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2533         bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2534         bhsr2t->bhsr2t_target_transfer_tag = htonl(target_transfer_tag);
2535         /*
2536          * XXX: Here we assume that cfiscsi_datamove() won't ever
2537          *      be running concurrently on several CPUs for a given
2538          *      command.
2539          */
2540         bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2541         PDU_R2TSN(request)++;
2542         /*
2543          * This is the offset within the current SCSI command;
2544          * i.e. for the first call of datamove(), it will be 0,
2545          * and for subsequent ones it will be the sum of lengths
2546          * of previous ones.
2547          *
2548          * The ext_data_filled is to account for unsolicited
2549          * (immediate) data that might have already arrived.
2550          */
2551         bhsr2t->bhsr2t_buffer_offset =
2552             htonl(io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled);
2553         /*
2554          * This is the total length (sum of S/G lengths) this call
2555          * to cfiscsi_datamove() is supposed to handle.
2556          *
2557          * XXX: Limit it to MaxBurstLength.
2558          */
2559         bhsr2t->bhsr2t_desired_data_transfer_length =
2560             htonl(io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
2561         cfiscsi_pdu_queue(response);
2562 }
2563
2564 static void
2565 cfiscsi_datamove(union ctl_io *io)
2566 {
2567
2568         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2569                 cfiscsi_datamove_in(io);
2570         else
2571                 cfiscsi_datamove_out(io);
2572 }
2573
2574 static void
2575 cfiscsi_scsi_command_done(union ctl_io *io)
2576 {
2577         struct icl_pdu *request, *response;
2578         struct iscsi_bhs_scsi_command *bhssc;
2579         struct iscsi_bhs_scsi_response *bhssr;
2580 #ifdef DIAGNOSTIC
2581         struct cfiscsi_data_wait *cdw;
2582 #endif
2583         struct cfiscsi_session *cs;
2584         uint16_t sense_length;
2585
2586         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2587         cs = PDU_SESSION(request);
2588         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2589         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2590             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2591             ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2592
2593         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2594         //    bhssc->bhssc_initiator_task_tag);
2595
2596 #ifdef DIAGNOSTIC
2597         CFISCSI_SESSION_LOCK(cs);
2598         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2599                 KASSERT(bhssc->bhssc_initiator_task_tag !=
2600                     cdw->cdw_initiator_task_tag, ("dangling cdw"));
2601         CFISCSI_SESSION_UNLOCK(cs);
2602 #endif
2603
2604         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2605         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2606         bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2607         bhssr->bhssr_flags = 0x80;
2608         /*
2609          * XXX: We don't deal with bidirectional under/overflows;
2610          *      does anything actually support those?
2611          */
2612         if (PDU_TOTAL_TRANSFER_LEN(request) <
2613             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2614                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2615                 bhssr->bhssr_residual_count =
2616                     htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2617                     PDU_TOTAL_TRANSFER_LEN(request));
2618                 //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2619                 //    ntohl(bhssr->bhssr_residual_count));
2620         } else if (PDU_TOTAL_TRANSFER_LEN(request) >
2621             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2622                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2623                 bhssr->bhssr_residual_count =
2624                     htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2625                     ntohl(bhssc->bhssc_expected_data_transfer_length));
2626                 //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2627                 //    ntohl(bhssr->bhssr_residual_count));
2628         }
2629         bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2630         bhssr->bhssr_status = io->scsiio.scsi_status;
2631         bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2632         bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2633
2634         if (io->scsiio.sense_len > 0) {
2635 #if 0
2636                 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2637                     io->scsiio.sense_len);
2638 #endif
2639                 sense_length = htons(io->scsiio.sense_len);
2640                 icl_pdu_append_data(response,
2641                     &sense_length, sizeof(sense_length), M_WAITOK);
2642                 icl_pdu_append_data(response,
2643                     &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2644         }
2645
2646         ctl_free_io(io);
2647         icl_pdu_free(request);
2648         cfiscsi_pdu_queue(response);
2649 }
2650
2651 static void
2652 cfiscsi_task_management_done(union ctl_io *io)
2653 {
2654         struct icl_pdu *request, *response;
2655         struct iscsi_bhs_task_management_request *bhstmr;
2656         struct iscsi_bhs_task_management_response *bhstmr2;
2657         struct cfiscsi_data_wait *cdw, *tmpcdw;
2658         struct cfiscsi_session *cs;
2659
2660         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2661         cs = PDU_SESSION(request);
2662         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2663         KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2664             ISCSI_BHS_OPCODE_TASK_REQUEST,
2665             ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2666
2667 #if 0
2668         CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2669             bhstmr->bhstmr_initiator_task_tag,
2670             bhstmr->bhstmr_referenced_task_tag);
2671 #endif
2672
2673         if ((bhstmr->bhstmr_function & ~0x80) ==
2674             BHSTMR_FUNCTION_ABORT_TASK) {
2675                 /*
2676                  * Make sure we no longer wait for Data-Out for this command.
2677                  */
2678                 CFISCSI_SESSION_LOCK(cs);
2679                 TAILQ_FOREACH_SAFE(cdw,
2680                     &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2681                         if (bhstmr->bhstmr_referenced_task_tag !=
2682                             cdw->cdw_initiator_task_tag)
2683                                 continue;
2684
2685 #if 0
2686                         CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2687                             "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2688 #endif
2689                         TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2690                             cdw, cdw_next);
2691                         cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2692                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2693                 }
2694                 CFISCSI_SESSION_UNLOCK(cs);
2695         }
2696
2697         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2698         bhstmr2 = (struct iscsi_bhs_task_management_response *)
2699             response->ip_bhs;
2700         bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2701         bhstmr2->bhstmr_flags = 0x80;
2702         if (io->io_hdr.status == CTL_SUCCESS) {
2703                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2704         } else {
2705                 /*
2706                  * XXX: How to figure out what exactly went wrong?  iSCSI spec
2707                  *      expects us to provide detailed error, e.g. "Task does
2708                  *      not exist" or "LUN does not exist".
2709                  */
2710                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED");
2711                 bhstmr2->bhstmr_response =
2712                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2713         }
2714         bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2715
2716         ctl_free_io(io);
2717         icl_pdu_free(request);
2718         cfiscsi_pdu_queue(response);
2719 }
2720
2721 static void
2722 cfiscsi_done(union ctl_io *io)
2723 {
2724         struct icl_pdu *request;
2725         struct cfiscsi_session *cs;
2726
2727         KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2728                 ("invalid CTL status %#x", io->io_hdr.status));
2729
2730         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2731         if (request == NULL) {
2732                 /*
2733                  * Implicit task termination has just completed; nothing to do.
2734                  */
2735                 return;
2736         }
2737
2738         cs = PDU_SESSION(request);
2739         refcount_release(&cs->cs_outstanding_ctl_pdus);
2740
2741         switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
2742         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
2743                 cfiscsi_scsi_command_done(io);
2744                 break;
2745         case ISCSI_BHS_OPCODE_TASK_REQUEST:
2746                 cfiscsi_task_management_done(io);
2747                 break;
2748         default:
2749                 panic("cfiscsi_done called with wrong opcode 0x%x",
2750                     request->ip_bhs->bhs_opcode);
2751         }
2752 }