sys/cam/ctl/ctl_frontend_iscsi.c

   1 /*-
   2  * Copyright (c) 2012 The FreeBSD Foundation
   3  * All rights reserved.
   4  *
   5  * This software was developed by Edward Tomasz Napierala under sponsorship
   6  * from the FreeBSD Foundation.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  *
  29  * $FreeBSD$
  30  */
  31
  32 /*
  33  * CTL frontend for the iSCSI protocol.
  34  */
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include <sys/param.h>
  40 #include <sys/capability.h>
  41 #include <sys/condvar.h>
  42 #include <sys/file.h>
  43 #include <sys/kernel.h>
  44 #include <sys/kthread.h>
  45 #include <sys/lock.h>
  46 #include <sys/malloc.h>
  47 #include <sys/module.h>
  48 #include <sys/mutex.h>
  49 #include <sys/queue.h>
  50 #include <sys/sbuf.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/systm.h>
  53 #include <sys/uio.h>
  54 #include <sys/unistd.h>
  55 #include <vm/uma.h>
  56
  57 #include <cam/scsi/scsi_all.h>
  58 #include <cam/scsi/scsi_da.h>
  59 #include <cam/ctl/ctl_io.h>
  60 #include <cam/ctl/ctl.h>
  61 #include <cam/ctl/ctl_backend.h>
  62 #include <cam/ctl/ctl_error.h>
  63 #include <cam/ctl/ctl_frontend.h>
  64 #include <cam/ctl/ctl_frontend_internal.h>
  65 #include <cam/ctl/ctl_debug.h>
  66 #include <cam/ctl/ctl_ha.h>
  67 #include <cam/ctl/ctl_ioctl.h>
  68 #include <cam/ctl/ctl_private.h>
  69
  70 #include "../../dev/iscsi/icl.h"
  71 #include "../../dev/iscsi/iscsi_proto.h"
  72 #include "ctl_frontend_iscsi.h"
  73
  74 #ifdef ICL_KERNEL_PROXY
  75 #include <sys/socketvar.h>
  76 #endif
  77
  78 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
  79 static uma_zone_t cfiscsi_data_wait_zone;
  80
  81 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD, 0,
  82     "CAM Target Layer iSCSI Frontend");
  83 static int debug = 3;
  84 TUNABLE_INT("kern.cam.ctl.iscsi.debug", &debug);
  85 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RW,
  86     &debug, 1, "Enable debug messages");
  87 static int ping_timeout = 5;
  88 TUNABLE_INT("kern.cam.ctl.iscsi.ping_timeout", &ping_timeout);
  89 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RW,
  90     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
  91 static int login_timeout = 60;
  92 TUNABLE_INT("kern.cam.ctl.iscsi.login_timeout", &login_timeout);
  93 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RW,
  94     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
  95 static int maxcmdsn_delta = 256;
  96 TUNABLE_INT("kern.cam.ctl.iscsi.maxcmdsn_delta", &maxcmdsn_delta);
  97 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RW,
  98     &maxcmdsn_delta, 256, "Number of commands the initiator can send "
  99     "without confirmation");
 100
 101 #define CFISCSI_DEBUG(X, ...)                                   \
 102         if (debug > 1) {                                        \
 103                 printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
 104         } while (0)
 105
 106 #define CFISCSI_WARN(X, ...)                                    \
 107         if (debug > 0) {                                        \
 108                 printf("WARNING: %s: " X "\n",                  \
 109                     __func__, ## __VA_ARGS__);                  \
 110         } while (0)
 111
 112 #define CFISCSI_SESSION_DEBUG(S, X, ...)                        \
 113         if (debug > 1) {                                        \
 114                 printf("%s: %s (%s): " X "\n",                  \
 115                     __func__, S->cs_initiator_addr,             \
 116                     S->cs_initiator_name, ## __VA_ARGS__);      \
 117         } while (0)
 118
 119 #define CFISCSI_SESSION_WARN(S, X, ...)                         \
 120         if (debug > 0) {                                        \
 121                 printf("WARNING: %s (%s): " X "\n",             \
 122                     S->cs_initiator_addr,                       \
 123                     S->cs_initiator_name, ## __VA_ARGS__);      \
 124         } while (0)
 125
 126 #define CFISCSI_SESSION_LOCK(X)         mtx_lock(&X->cs_lock)
 127 #define CFISCSI_SESSION_UNLOCK(X)       mtx_unlock(&X->cs_lock)
 128 #define CFISCSI_SESSION_LOCK_ASSERT(X)  mtx_assert(&X->cs_lock, MA_OWNED)
 129
 130 #define CONN_SESSION(X)                 ((struct cfiscsi_session *)(X)->ic_prv0)
 131 #define PDU_SESSION(X)                  CONN_SESSION((X)->ip_conn)
 132 #define PDU_EXPDATASN(X)                (X)->ip_prv0
 133 #define PDU_TOTAL_TRANSFER_LEN(X)       (X)->ip_prv1
 134 #define PDU_R2TSN(X)                    (X)->ip_prv2
 135
 136 int             cfiscsi_init(void);
 137 static void     cfiscsi_online(void *arg);
 138 static void     cfiscsi_offline(void *arg);
 139 static int      cfiscsi_targ_enable(void *arg, struct ctl_id targ_id);
 140 static int      cfiscsi_targ_disable(void *arg, struct ctl_id targ_id);
 141 static int      cfiscsi_lun_enable(void *arg,
 142                     struct ctl_id target_id, int lun_id);
 143 static int      cfiscsi_lun_disable(void *arg,
 144                     struct ctl_id target_id, int lun_id);
 145 static int      cfiscsi_ioctl(struct cdev *dev,
 146                     u_long cmd, caddr_t addr, int flag, struct thread *td);
 147 static int      cfiscsi_devid(struct ctl_scsiio *ctsio, int alloc_len);
 148 static void     cfiscsi_datamove(union ctl_io *io);
 149 static void     cfiscsi_done(union ctl_io *io);
 150 static uint32_t cfiscsi_map_lun(void *arg, uint32_t lun);
 151 static bool     cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
 152 static void     cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
 153 static void     cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
 154 static void     cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
 155 static void     cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
 156 static void     cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
 157 static void     cfiscsi_session_terminate(struct cfiscsi_session *cs);
 158 static struct cfiscsi_target    *cfiscsi_target_find(struct cfiscsi_softc
 159                     *softc, const char *name);
 160 static void     cfiscsi_target_release(struct cfiscsi_target *ct);
 161 static void     cfiscsi_session_delete(struct cfiscsi_session *cs);
 162
 163 static struct cfiscsi_softc cfiscsi_softc;
 164 extern struct ctl_softc *control_softc;
 165
 166 static int cfiscsi_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 167
 168 static moduledata_t cfiscsi_moduledata = {
 169         "ctlcfiscsi",
 170         cfiscsi_module_event_handler,
 171         NULL
 172 };
 173
 174 DECLARE_MODULE(ctlcfiscsi, cfiscsi_moduledata, SI_SUB_CONFIGURE, SI_ORDER_FOURTH);
 175 MODULE_VERSION(ctlcfiscsi, 1);
 176 MODULE_DEPEND(ctlcfiscsi, ctl, 1, 1, 1);
 177 MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1);
 178
 179 static struct icl_pdu *
 180 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
 181 {
 182
 183         return (icl_pdu_new_bhs(request->ip_conn, flags));
 184 }
 185
 186 static bool
 187 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
 188 {
 189         const struct iscsi_bhs_scsi_command *bhssc;
 190         struct cfiscsi_session *cs;
 191         uint32_t cmdsn, expstatsn;
 192
 193         cs = PDU_SESSION(request);
 194
 195         /*
 196          * Every incoming PDU - not just NOP-Out - resets the ping timer.
 197          * The purpose of the timeout is to reset the connection when it stalls;
 198          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
 199          * in some queue.
 200          *
 201          * XXX: Locking?
 202          */
 203         cs->cs_timeout = 0;
 204
 205         /*
 206          * Data-Out PDUs don't contain CmdSN.
 207          */
 208         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 209             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 210                 return (false);
 211
 212         /*
 213          * We're only using fields common for all the request
 214          * (initiator -> target) PDUs.
 215          */
 216         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
 217         cmdsn = ntohl(bhssc->bhssc_cmdsn);
 218         expstatsn = ntohl(bhssc->bhssc_expstatsn);
 219
 220         CFISCSI_SESSION_LOCK(cs);
 221 #if 0
 222         if (expstatsn != cs->cs_statsn) {
 223                 CFISCSI_SESSION_DEBUG(cs, "received PDU with ExpStatSN %d, "
 224                     "while current StatSN is %d", expstatsn,
 225                     cs->cs_statsn);
 226         }
 227 #endif
 228
 229         /*
 230          * The target MUST silently ignore any non-immediate command outside
 231          * of this range.
 232          */
 233         if (cmdsn < cs->cs_cmdsn || cmdsn > cs->cs_cmdsn + maxcmdsn_delta) {
 234                 CFISCSI_SESSION_UNLOCK(cs);
 235                 CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %d, "
 236                     "while expected CmdSN was %d", cmdsn, cs->cs_cmdsn);
 237                 return (true);
 238         }
 239
 240         if ((request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
 241                 cs->cs_cmdsn++;
 242
 243         CFISCSI_SESSION_UNLOCK(cs);
 244
 245         return (false);
 246 }
 247
 248 static void
 249 cfiscsi_pdu_handle(struct icl_pdu *request)
 250 {
 251         struct cfiscsi_session *cs;
 252         bool ignore;
 253
 254         cs = PDU_SESSION(request);
 255
 256         ignore = cfiscsi_pdu_update_cmdsn(request);
 257         if (ignore) {
 258                 icl_pdu_free(request);
 259                 return;
 260         }
 261
 262         /*
 263          * Handle the PDU; this includes e.g. receiving the remaining
 264          * part of PDU and submitting the SCSI command to CTL
 265          * or queueing a reply.  The handling routine is responsible
 266          * for freeing the PDU when it's no longer needed.
 267          */
 268         switch (request->ip_bhs->bhs_opcode &
 269             ~ISCSI_BHS_OPCODE_IMMEDIATE) {
 270         case ISCSI_BHS_OPCODE_NOP_OUT:
 271                 cfiscsi_pdu_handle_nop_out(request);
 272                 break;
 273         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
 274                 cfiscsi_pdu_handle_scsi_command(request);
 275                 break;
 276         case ISCSI_BHS_OPCODE_TASK_REQUEST:
 277                 cfiscsi_pdu_handle_task_request(request);
 278                 break;
 279         case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
 280                 cfiscsi_pdu_handle_data_out(request);
 281                 break;
 282         case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
 283                 cfiscsi_pdu_handle_logout_request(request);
 284                 break;
 285         default:
 286                 CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
 287                     "opcode 0x%x; dropping connection",
 288                     request->ip_bhs->bhs_opcode);
 289                 icl_pdu_free(request);
 290                 cfiscsi_session_terminate(cs);
 291         }
 292
 293 }
 294
 295 static void
 296 cfiscsi_receive_callback(struct icl_pdu *request)
 297 {
 298         struct cfiscsi_session *cs;
 299
 300         cs = PDU_SESSION(request);
 301
 302 #ifdef ICL_KERNEL_PROXY
 303         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 304                 if (cs->cs_login_pdu == NULL)
 305                         cs->cs_login_pdu = request;
 306                 else
 307                         icl_pdu_free(request);
 308                 cv_signal(&cs->cs_login_cv);
 309                 return;
 310         }
 311 #endif
 312
 313         cfiscsi_pdu_handle(request);
 314 }
 315
 316 static void
 317 cfiscsi_error_callback(struct icl_conn *ic)
 318 {
 319         struct cfiscsi_session *cs;
 320
 321         cs = CONN_SESSION(ic);
 322
 323         CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
 324         cfiscsi_session_terminate(cs);
 325 }
 326
 327 static int
 328 cfiscsi_pdu_prepare(struct icl_pdu *response)
 329 {
 330         struct cfiscsi_session *cs;
 331         struct iscsi_bhs_scsi_response *bhssr;
 332         bool advance_statsn = true;
 333
 334         cs = PDU_SESSION(response);
 335
 336         CFISCSI_SESSION_LOCK_ASSERT(cs);
 337
 338         /*
 339          * We're only using fields common for all the response
 340          * (target -> initiator) PDUs.
 341          */
 342         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
 343
 344         /*
 345          * 10.8.3: "The StatSN for this connection is not advanced
 346          * after this PDU is sent."
 347          */
 348         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
 349                 advance_statsn = false;
 350
 351         /*
 352          * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
 353          * StatSN for the connection is not advanced after this PDU is sent."
 354          */
 355         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN &&
 356             bhssr->bhssr_initiator_task_tag == 0xffffffff)
 357                 advance_statsn = false;
 358
 359         /*
 360          * See the comment below - StatSN is not meaningful and must
 361          * not be advanced.
 362          */
 363         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN)
 364                 advance_statsn = false;
 365
 366         /*
 367          * 10.7.3: "The fields StatSN, Status, and Residual Count
 368          * only have meaningful content if the S bit is set to 1."
 369          */
 370         if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN)
 371                 bhssr->bhssr_statsn = htonl(cs->cs_statsn);
 372         bhssr->bhssr_expcmdsn = htonl(cs->cs_cmdsn);
 373         bhssr->bhssr_maxcmdsn = htonl(cs->cs_cmdsn + maxcmdsn_delta);
 374
 375         if (advance_statsn)
 376                 cs->cs_statsn++;
 377
 378         return (0);
 379 }
 380
 381 static void
 382 cfiscsi_pdu_queue(struct icl_pdu *response)
 383 {
 384         struct cfiscsi_session *cs;
 385
 386         cs = PDU_SESSION(response);
 387
 388         CFISCSI_SESSION_LOCK(cs);
 389         cfiscsi_pdu_prepare(response);
 390         icl_pdu_queue(response);
 391         CFISCSI_SESSION_UNLOCK(cs);
 392 }
 393
 394 static uint32_t
 395 cfiscsi_decode_lun(uint64_t encoded)
 396 {
 397         uint8_t lun[8];
 398         uint32_t result;
 399
 400         /*
 401          * The LUN field in iSCSI PDUs may look like an ordinary 64 bit number,
 402          * but is in fact an evil, multidimensional structure defined
 403          * in SCSI Architecture Model 5 (SAM-5), section 4.6.
 404          */
 405         memcpy(lun, &encoded, sizeof(lun));
 406         switch (lun[0] & 0xC0) {
 407         case 0x00:
 408                 if ((lun[0] & 0x3f) != 0 || lun[2] != 0 || lun[3] != 0 ||
 409                     lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) {
 410                         CFISCSI_WARN("malformed LUN "
 411                             "(peripheral device addressing method): 0x%jx",
 412                             (uintmax_t)encoded);
 413                         result = 0xffffffff;
 414                         break;
 415                 }
 416                 result = lun[1];
 417                 break;
 418         case 0x40:
 419                 if (lun[2] != 0 || lun[3] != 0 || lun[4] != 0 || lun[5] != 0 ||
 420                     lun[6] != 0 || lun[7] != 0) {
 421                         CFISCSI_WARN("malformed LUN "
 422                             "(flat address space addressing method): 0x%jx",
 423                             (uintmax_t)encoded);
 424                         result = 0xffffffff;
 425                         break;
 426                 }
 427                 result = ((lun[0] & 0x3f) << 8) + lun[1];
 428                 break;
 429         case 0xC0:
 430                 if (lun[0] != 0xD2 || lun[4] != 0 || lun[5] != 0 ||
 431                     lun[6] != 0 || lun[7] != 0) {
 432                         CFISCSI_WARN("malformed LUN (extended flat "
 433                             "address space addressing method): 0x%jx",
 434                             (uintmax_t)encoded);
 435                         result = 0xffffffff;
 436                         break;
 437                 }
 438                 result = (lun[1] << 16) + (lun[2] << 8) + lun[3];
 439         default:
 440                 CFISCSI_WARN("unsupported LUN format 0x%jx",
 441                     (uintmax_t)encoded);
 442                 result = 0xffffffff;
 443                 break;
 444         }
 445
 446         return (result);
 447 }
 448
 449 static void
 450 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
 451 {
 452         struct cfiscsi_session *cs;
 453         struct iscsi_bhs_nop_out *bhsno;
 454         struct iscsi_bhs_nop_in *bhsni;
 455         struct icl_pdu *response;
 456         void *data = NULL;
 457         size_t datasize;
 458         int error;
 459
 460         cs = PDU_SESSION(request);
 461         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
 462
 463         if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
 464                 /*
 465                  * Nothing to do, iscsi_pdu_update_statsn() already
 466                  * zeroed the timeout.
 467                  */
 468                 icl_pdu_free(request);
 469                 return;
 470         }
 471
 472         datasize = icl_pdu_data_segment_length(request);
 473         if (datasize > 0) {
 474                 data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
 475                 if (data == NULL) {
 476                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 477                             "dropping connection");
 478                         icl_pdu_free(request);
 479                         cfiscsi_session_terminate(cs);
 480                         return;
 481                 }
 482                 icl_pdu_get_data(request, 0, data, datasize);
 483         }
 484
 485         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 486         if (response == NULL) {
 487                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 488                     "droppping connection");
 489                 free(data, M_CFISCSI);
 490                 icl_pdu_free(request);
 491                 cfiscsi_session_terminate(cs);
 492                 return;
 493         }
 494         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
 495         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
 496         bhsni->bhsni_flags = 0x80;
 497         bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
 498         bhsni->bhsni_target_transfer_tag = 0xffffffff;
 499         if (datasize > 0) {
 500                 error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
 501                 if (error != 0) {
 502                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 503                             "dropping connection");
 504                         free(data, M_CFISCSI);
 505                         icl_pdu_free(request);
 506                         icl_pdu_free(response);
 507                         cfiscsi_session_terminate(cs);
 508                         return;
 509                 }
 510                 free(data, M_CFISCSI);
 511         }
 512
 513         icl_pdu_free(request);
 514         cfiscsi_pdu_queue(response);
 515 }
 516
 517 static void
 518 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
 519 {
 520         struct iscsi_bhs_scsi_command *bhssc;
 521         struct cfiscsi_session *cs;
 522         union ctl_io *io;
 523         int error;
 524
 525         cs = PDU_SESSION(request);
 526         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
 527         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
 528         //    bhssc->bhssc_initiator_task_tag);
 529
 530         if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
 531                 CFISCSI_SESSION_WARN(cs, "unsolicited data with "
 532                     "ImmediateData=No; dropping connection");
 533                 icl_pdu_free(request);
 534                 cfiscsi_session_terminate(cs);
 535                 return;
 536         }
 537         io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
 538         if (io == NULL) {
 539                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io; "
 540                     "dropping connection");
 541                 icl_pdu_free(request);
 542                 cfiscsi_session_terminate(cs);
 543                 return;
 544         }
 545         ctl_zero_io(io);
 546         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 547         io->io_hdr.io_type = CTL_IO_SCSI;
 548         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 549         io->io_hdr.nexus.targ_port = cs->cs_target->ct_softc->fe.targ_port;
 550         io->io_hdr.nexus.targ_target.id = 0;
 551         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhssc->bhssc_lun);
 552         io->io_hdr.nexus.lun_map_fn = cfiscsi_map_lun;
 553         io->io_hdr.nexus.lun_map_arg = cs;
 554         io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
 555         switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
 556         case BHSSC_FLAGS_ATTR_UNTAGGED:
 557                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 558                 break;
 559         case BHSSC_FLAGS_ATTR_SIMPLE:
 560                 io->scsiio.tag_type = CTL_TAG_SIMPLE;
 561                 break;
 562         case BHSSC_FLAGS_ATTR_ORDERED:
 563                 io->scsiio.tag_type = CTL_TAG_ORDERED;
 564                 break;
 565         case BHSSC_FLAGS_ATTR_HOQ:
 566                 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
 567                 break;
 568         case BHSSC_FLAGS_ATTR_ACA:
 569                 io->scsiio.tag_type = CTL_TAG_ACA;
 570                 break;
 571         default:
 572                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
 573                 CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
 574                     bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
 575                 break;
 576         }
 577         io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
 578         memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
 579         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 580         error = ctl_queue(io);
 581         if (error != CTL_RETVAL_COMPLETE) {
 582                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 583                     "dropping connection", error);
 584                 ctl_free_io(io);
 585                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 586                 icl_pdu_free(request);
 587                 cfiscsi_session_terminate(cs);
 588         }
 589 }
 590
 591 static void
 592 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
 593 {
 594         struct iscsi_bhs_task_management_request *bhstmr;
 595         struct iscsi_bhs_task_management_response *bhstmr2;
 596         struct icl_pdu *response;
 597         struct cfiscsi_session *cs;
 598         union ctl_io *io;
 599         int error;
 600
 601         cs = PDU_SESSION(request);
 602         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
 603         io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
 604         if (io == NULL) {
 605                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io;"
 606                     "dropping connection");
 607                 icl_pdu_free(request);
 608                 cfiscsi_session_terminate(cs);
 609                 return;
 610         }
 611         ctl_zero_io(io);
 612         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = request;
 613         io->io_hdr.io_type = CTL_IO_TASK;
 614         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
 615         io->io_hdr.nexus.targ_port = cs->cs_target->ct_softc->fe.targ_port;
 616         io->io_hdr.nexus.targ_target.id = 0;
 617         io->io_hdr.nexus.targ_lun = cfiscsi_decode_lun(bhstmr->bhstmr_lun);
 618         io->io_hdr.nexus.lun_map_fn = cfiscsi_map_lun;
 619         io->io_hdr.nexus.lun_map_arg = cs;
 620         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
 621
 622         switch (bhstmr->bhstmr_function & ~0x80) {
 623         case BHSTMR_FUNCTION_ABORT_TASK:
 624 #if 0
 625                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
 626 #endif
 627                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
 628                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
 629                 break;
 630         case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
 631 #if 0
 632                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
 633 #endif
 634                 io->taskio.task_action = CTL_TASK_LUN_RESET;
 635                 break;
 636         case BHSTMR_FUNCTION_TARGET_WARM_RESET:
 637 #if 0
 638                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
 639 #endif
 640                 io->taskio.task_action = CTL_TASK_TARGET_RESET;
 641                 break;
 642         default:
 643                 CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
 644                     bhstmr->bhstmr_function & ~0x80);
 645                 ctl_free_io(io);
 646
 647                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 648                 if (response == NULL) {
 649                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
 650                             "dropping connection");
 651                         icl_pdu_free(request);
 652                         cfiscsi_session_terminate(cs);
 653                         return;
 654                 }
 655                 bhstmr2 = (struct iscsi_bhs_task_management_response *)
 656                     response->ip_bhs;
 657                 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
 658                 bhstmr2->bhstmr_flags = 0x80;
 659                 bhstmr2->bhstmr_response =
 660                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
 661                 bhstmr2->bhstmr_initiator_task_tag =
 662                     bhstmr->bhstmr_initiator_task_tag;
 663                 icl_pdu_free(request);
 664                 cfiscsi_pdu_queue(response);
 665                 return;
 666         }
 667
 668         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
 669         error = ctl_queue(io);
 670         if (error != CTL_RETVAL_COMPLETE) {
 671                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d; "
 672                     "dropping connection", error);
 673                 ctl_free_io(io);
 674                 refcount_release(&cs->cs_outstanding_ctl_pdus);
 675                 icl_pdu_free(request);
 676                 cfiscsi_session_terminate(cs);
 677         }
 678 }
 679
 680 static bool
 681 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
 682 {
 683         struct iscsi_bhs_data_out *bhsdo;
 684         struct cfiscsi_session *cs;
 685         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
 686         size_t copy_len, len, off, buffer_offset;
 687         int ctl_sg_count;
 688         union ctl_io *io;
 689
 690         cs = PDU_SESSION(request);
 691
 692         KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 693             ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
 694             (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 695             ISCSI_BHS_OPCODE_SCSI_COMMAND,
 696             ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
 697
 698         /*
 699          * We're only using fields common for Data Out and SCSI Command PDUs.
 700          */
 701         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 702
 703         io = cdw->cdw_ctl_io;
 704         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 705             ("CTL_FLAG_DATA_IN"));
 706
 707 #if 0
 708         CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
 709             request->ip_data_len, io->scsiio.kern_total_len);
 710 #endif
 711
 712         if (io->scsiio.kern_sg_entries > 0) {
 713                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
 714                 ctl_sg_count = io->scsiio.kern_sg_entries;
 715         } else {
 716                 ctl_sglist = &ctl_sg_entry;
 717                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
 718                 ctl_sglist->len = io->scsiio.kern_data_len;
 719                 ctl_sg_count = 1;
 720         }
 721
 722         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 723             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
 724                 buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
 725         else
 726                 buffer_offset = 0;
 727
 728         /*
 729          * Make sure the offset, as sent by the initiator, matches the offset
 730          * we're supposed to be at in the scatter-gather list.
 731          */
 732         if (buffer_offset != io->scsiio.ext_data_filled) {
 733                 CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
 734                     "expected %zd", buffer_offset,
 735                     (size_t)io->scsiio.ext_data_filled);
 736                 cfiscsi_session_terminate(cs);
 737                 return (true);
 738         }
 739
 740         /*
 741          * This is the offset within the PDU data segment, as opposed
 742          * to buffer_offset, which is the offset within the task (SCSI
 743          * command).
 744          */
 745         off = 0;
 746         len = icl_pdu_data_segment_length(request);
 747
 748         /*
 749          * Iterate over the scatter/gather segments, filling them with data
 750          * from the PDU data segment.  Note that this can get called multiple
 751          * times for one SCSI command; the cdw structure holds state for the
 752          * scatter/gather list.
 753          */
 754         for (;;) {
 755                 KASSERT(cdw->cdw_sg_index < ctl_sg_count,
 756                     ("cdw->cdw_sg_index >= ctl_sg_count"));
 757                 if (cdw->cdw_sg_len == 0) {
 758                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
 759                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
 760                 }
 761                 KASSERT(off <= len, ("len > off"));
 762                 copy_len = len - off;
 763                 if (copy_len > cdw->cdw_sg_len)
 764                         copy_len = cdw->cdw_sg_len;
 765
 766                 icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
 767                 cdw->cdw_sg_addr += copy_len;
 768                 cdw->cdw_sg_len -= copy_len;
 769                 off += copy_len;
 770                 io->scsiio.ext_data_filled += copy_len;
 771
 772                 if (cdw->cdw_sg_len == 0) {
 773                         /*
 774                          * End of current segment.
 775                          */
 776                         if (cdw->cdw_sg_index == ctl_sg_count - 1) {
 777                                 /*
 778                                  * Last segment in scatter/gather list.
 779                                  */
 780                                 break;
 781                         }
 782                         cdw->cdw_sg_index++;
 783                 }
 784
 785                 if (off == len) {
 786                         /*
 787                          * End of PDU payload.
 788                          */
 789                         break;
 790                 }
 791         }
 792
 793         if (len > off) {
 794                 CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
 795                     "expected %zd", icl_pdu_data_segment_length(request), off);
 796                 cfiscsi_session_terminate(cs);
 797                 return (true);
 798         }
 799
 800         if (bhsdo->bhsdo_flags & BHSDO_FLAGS_F ||
 801             io->scsiio.ext_data_filled == io->scsiio.kern_total_len) {
 802                 if ((bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
 803                         CFISCSI_SESSION_WARN(cs, "got the final packet without "
 804                             "the F flag; flags = 0x%x; dropping connection",
 805                             bhsdo->bhsdo_flags);
 806                         cfiscsi_session_terminate(cs);
 807                         return (true);
 808                 }
 809
 810                 if (io->scsiio.ext_data_filled != io->scsiio.kern_total_len) {
 811                         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
 812                             ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
 813                                 CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
 814                                     "transmitted size was %zd bytes instead of %d; "
 815                                     "dropping connection",
 816                                     (size_t)io->scsiio.ext_data_filled,
 817                                     io->scsiio.kern_total_len);
 818                                 cfiscsi_session_terminate(cs);
 819                                 return (true);
 820                         } else {
 821                                 /*
 822                                  * For SCSI Command PDU, this just means we need to
 823                                  * solicit more data by sending R2T.
 824                                  */
 825                                 return (false);
 826                         }
 827                 }
 828 #if 0
 829                 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
 830                     "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
 831 #endif
 832
 833                 return (true);
 834         }
 835
 836         return (false);
 837 }
 838
 839 static void
 840 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
 841 {
 842         struct iscsi_bhs_data_out *bhsdo;
 843         struct cfiscsi_session *cs;
 844         struct cfiscsi_data_wait *cdw = NULL;
 845         union ctl_io *io;
 846         bool done;
 847
 848         cs = PDU_SESSION(request);
 849         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
 850
 851         CFISCSI_SESSION_LOCK(cs);
 852         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
 853 #if 0
 854                 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
 855                     "ttt 0x%x, itt 0x%x",
 856                     bhsdo->bhsdo_target_transfer_tag,
 857                     bhsdo->bhsdo_initiator_task_tag,
 858                     cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
 859 #endif
 860                 if (bhsdo->bhsdo_target_transfer_tag ==
 861                     cdw->cdw_target_transfer_tag)
 862                         break;
 863         }
 864         CFISCSI_SESSION_UNLOCK(cs);
 865         if (cdw == NULL) {
 866                 CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
 867                     "0x%x, not found; dropping connection",
 868                     bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
 869                 icl_pdu_free(request);
 870                 cfiscsi_session_terminate(cs);
 871                 return;
 872         }
 873
 874         io = cdw->cdw_ctl_io;
 875         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
 876             ("CTL_FLAG_DATA_IN"));
 877
 878         done = cfiscsi_handle_data_segment(request, cdw);
 879         if (done) {
 880                 CFISCSI_SESSION_LOCK(cs);
 881                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
 882                 CFISCSI_SESSION_UNLOCK(cs);
 883                 uma_zfree(cfiscsi_data_wait_zone, cdw);
 884                 io->scsiio.be_move_done(io);
 885         }
 886
 887         icl_pdu_free(request);
 888 }
 889
 890 static void
 891 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
 892 {
 893         struct iscsi_bhs_logout_request *bhslr;
 894         struct iscsi_bhs_logout_response *bhslr2;
 895         struct icl_pdu *response;
 896         struct cfiscsi_session *cs;
 897
 898         cs = PDU_SESSION(request);
 899         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
 900         switch (bhslr->bhslr_reason & 0x7f) {
 901         case BHSLR_REASON_CLOSE_SESSION:
 902         case BHSLR_REASON_CLOSE_CONNECTION:
 903                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 904                 if (response == NULL) {
 905                         CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
 906                         icl_pdu_free(request);
 907                         cfiscsi_session_terminate(cs);
 908                         return;
 909                 }
 910                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 911                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 912                 bhslr2->bhslr_flags = 0x80;
 913                 bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
 914                 bhslr2->bhslr_initiator_task_tag =
 915                     bhslr->bhslr_initiator_task_tag;
 916                 icl_pdu_free(request);
 917                 cfiscsi_pdu_queue(response);
 918                 cfiscsi_session_terminate(cs);
 919                 break;
 920         case BHSLR_REASON_REMOVE_FOR_RECOVERY:
 921                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
 922                 if (response == NULL) {
 923                         CFISCSI_SESSION_WARN(cs,
 924                             "failed to allocate memory; dropping connection");
 925                         icl_pdu_free(request);
 926                         cfiscsi_session_terminate(cs);
 927                         return;
 928                 }
 929                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
 930                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
 931                 bhslr2->bhslr_flags = 0x80;
 932                 bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
 933                 bhslr2->bhslr_initiator_task_tag =
 934                     bhslr->bhslr_initiator_task_tag;
 935                 icl_pdu_free(request);
 936                 cfiscsi_pdu_queue(response);
 937                 break;
 938         default:
 939                 CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
 940                     bhslr->bhslr_reason);
 941                 icl_pdu_free(request);
 942                 cfiscsi_session_terminate(cs);
 943                 break;
 944         }
 945 }
 946
 947 static void
 948 cfiscsi_callout(void *context)
 949 {
 950         struct icl_pdu *cp;
 951         struct iscsi_bhs_nop_in *bhsni;
 952         struct cfiscsi_session *cs;
 953
 954         cs = context;
 955
 956         if (cs->cs_terminating)
 957                 return;
 958
 959         callout_schedule(&cs->cs_callout, 1 * hz);
 960
 961         CFISCSI_SESSION_LOCK(cs);
 962         cs->cs_timeout++;
 963         CFISCSI_SESSION_UNLOCK(cs);
 964
 965 #ifdef ICL_KERNEL_PROXY
 966         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
 967                 if (cs->cs_timeout > login_timeout) {
 968                         CFISCSI_SESSION_WARN(cs, "login timed out after "
 969                             "%d seconds; dropping connection", cs->cs_timeout);
 970                         cfiscsi_session_terminate(cs);
 971                 }
 972                 return;
 973         }
 974 #endif
 975
 976         if (cs->cs_timeout >= ping_timeout) {
 977                 CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
 978                     "dropping connection",  ping_timeout);
 979                 cfiscsi_session_terminate(cs);
 980                 return;
 981         }
 982
 983         /*
 984          * If the ping was reset less than one second ago - which means
 985          * that we've received some PDU during the last second - assume
 986          * the traffic flows correctly and don't bother sending a NOP-Out.
 987          *
 988          * (It's 2 - one for one second, and one for incrementing is_timeout
 989          * earlier in this routine.)
 990          */
 991         if (cs->cs_timeout < 2)
 992                 return;
 993
 994         cp = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
 995         if (cp == NULL) {
 996                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
 997                 return;
 998         }
 999         bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1000         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1001         bhsni->bhsni_flags = 0x80;
1002         bhsni->bhsni_initiator_task_tag = 0xffffffff;
1003
1004         cfiscsi_pdu_queue(cp);
1005 }
1006
1007 static void
1008 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1009 {
1010         struct cfiscsi_data_wait *cdw, *tmpcdw;
1011         union ctl_io *io;
1012         int error;
1013
1014 #ifdef notyet
1015         io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
1016         if (io == NULL) {
1017                 CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io");
1018                 return;
1019         }
1020         ctl_zero_io(io);
1021         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = NULL;
1022         io->io_hdr.io_type = CTL_IO_TASK;
1023         io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1024         io->io_hdr.nexus.targ_port = cs->cs_target->ct_softc->fe.targ_port;
1025         io->io_hdr.nexus.targ_target.id = 0;
1026         io->io_hdr.nexus.targ_lun = lun;
1027         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1028         io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
1029         error = ctl_queue(io);
1030         if (error != CTL_RETVAL_COMPLETE) {
1031                 CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1032                 ctl_free_io(io);
1033         }
1034 #else
1035         /*
1036          * CTL doesn't currently support CTL_TASK_ABORT_TASK_SET, so instead
1037          * just iterate over tasks that are waiting for something - data - and
1038          * terminate those.
1039          */
1040         CFISCSI_SESSION_LOCK(cs);
1041         TAILQ_FOREACH_SAFE(cdw,
1042             &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
1043                 io = ctl_alloc_io(cs->cs_target->ct_softc->fe.ctl_pool_ref);
1044                 if (io == NULL) {
1045                         CFISCSI_SESSION_WARN(cs, "can't allocate ctl_io");
1046                         return;
1047                 }
1048                 ctl_zero_io(io);
1049                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = NULL;
1050                 io->io_hdr.io_type = CTL_IO_TASK;
1051                 io->io_hdr.nexus.initid.id = cs->cs_ctl_initid;
1052                 io->io_hdr.nexus.targ_port =
1053                     cs->cs_target->ct_softc->fe.targ_port;
1054                 io->io_hdr.nexus.targ_target.id = 0;
1055                 //io->io_hdr.nexus.targ_lun = lun; /* Not needed? */
1056                 io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1057                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
1058                 io->taskio.tag_num = cdw->cdw_initiator_task_tag;
1059                 error = ctl_queue(io);
1060                 if (error != CTL_RETVAL_COMPLETE) {
1061                         CFISCSI_SESSION_WARN(cs, "ctl_queue() failed; error %d", error);
1062                         ctl_free_io(io);
1063                         return;
1064                 }
1065 #if 0
1066                 CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task tag "
1067                     "0x%x", cdw->cdw_initiator_task_tag);
1068 #endif
1069                 cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
1070                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1071                 uma_zfree(cfiscsi_data_wait_zone, cdw);
1072         }
1073         CFISCSI_SESSION_UNLOCK(cs);
1074 #endif
1075 }
1076
1077 static void
1078 cfiscsi_maintenance_thread(void *arg)
1079 {
1080         struct cfiscsi_session *cs;
1081
1082         cs = arg;
1083
1084         for (;;) {
1085                 CFISCSI_SESSION_LOCK(cs);
1086                 if (cs->cs_terminating == false)
1087                         cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1088                 CFISCSI_SESSION_UNLOCK(cs);
1089
1090                 if (cs->cs_terminating) {
1091                         cfiscsi_session_terminate_tasks(cs);
1092                         callout_drain(&cs->cs_callout);
1093
1094                         icl_conn_shutdown(cs->cs_conn);
1095                         icl_conn_close(cs->cs_conn);
1096
1097                         cs->cs_terminating++;
1098
1099                         /*
1100                          * XXX: We used to wait up to 30 seconds to deliver queued PDUs
1101                          *      to the initiator.  We also tried hard to deliver SCSI Responses
1102                          *      for the aborted PDUs.  We don't do that anymore.  We might need
1103                          *      to revisit that.
1104                          */
1105
1106                         cfiscsi_session_delete(cs);
1107                         kthread_exit();
1108                         return;
1109                 }
1110                 CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1111         }
1112 }
1113
1114 static void
1115 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1116 {
1117
1118         if (cs->cs_terminating != 0)
1119                 return;
1120         cs->cs_terminating = 1;
1121         cv_signal(&cs->cs_maintenance_cv);
1122 }
1123
1124 static int
1125 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1126 {
1127         int error, i;
1128         struct cfiscsi_softc *softc;
1129
1130         KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1131
1132         softc = &cfiscsi_softc;
1133
1134         mtx_lock(&softc->lock);
1135         for (i = 0; i < softc->max_initiators; i++) {
1136                 if (softc->ctl_initids[i] == 0)
1137                         break;
1138         }
1139         if (i == softc->max_initiators) {
1140                 CFISCSI_SESSION_WARN(cs, "too many concurrent sessions (%d)",
1141                     softc->max_initiators);
1142                 mtx_unlock(&softc->lock);
1143                 return (1);
1144         }
1145         softc->ctl_initids[i] = 1;
1146         mtx_unlock(&softc->lock);
1147
1148 #if 0
1149         CFISCSI_SESSION_DEBUG(cs, "adding initiator id %d, max %d",
1150             i, softc->max_initiators);
1151 #endif
1152         cs->cs_ctl_initid = i;
1153         error = ctl_add_initiator(0x0, softc->fe.targ_port, cs->cs_ctl_initid);
1154         if (error != 0) {
1155                 CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d", error);
1156                 mtx_lock(&softc->lock);
1157                 softc->ctl_initids[cs->cs_ctl_initid] = 0;
1158                 mtx_unlock(&softc->lock);
1159                 cs->cs_ctl_initid = -1;
1160                 return (1);
1161         }
1162
1163         return (0);
1164 }
1165
1166 static void
1167 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1168 {
1169         int error;
1170         struct cfiscsi_softc *softc;
1171
1172         if (cs->cs_ctl_initid == -1)
1173                 return;
1174
1175         softc = &cfiscsi_softc;
1176
1177         error = ctl_remove_initiator(softc->fe.targ_port, cs->cs_ctl_initid);
1178         if (error != 0) {
1179                 CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1180                     error);
1181         }
1182         mtx_lock(&softc->lock);
1183         softc->ctl_initids[cs->cs_ctl_initid] = 0;
1184         mtx_unlock(&softc->lock);
1185         cs->cs_ctl_initid = -1;
1186 }
1187
1188 static struct cfiscsi_session *
1189 cfiscsi_session_new(struct cfiscsi_softc *softc)
1190 {
1191         struct cfiscsi_session *cs;
1192         int error;
1193
1194         cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1195         if (cs == NULL) {
1196                 CFISCSI_WARN("malloc failed");
1197                 return (NULL);
1198         }
1199         cs->cs_ctl_initid = -1;
1200
1201         refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1202         TAILQ_INIT(&cs->cs_waiting_for_data_out);
1203         mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1204         cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1205 #ifdef ICL_KERNEL_PROXY
1206         cv_init(&cs->cs_login_cv, "cfiscsi_login");
1207 #endif
1208
1209         cs->cs_conn = icl_conn_new();
1210         cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1211         cs->cs_conn->ic_error = cfiscsi_error_callback;
1212         cs->cs_conn->ic_prv0 = cs;
1213
1214         error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1215         if (error != 0) {
1216                 CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1217                 free(cs, M_CFISCSI);
1218                 return (NULL);
1219         }
1220
1221         mtx_lock(&softc->lock);
1222         cs->cs_id = softc->last_session_id + 1;
1223         softc->last_session_id++;
1224         mtx_unlock(&softc->lock);
1225
1226         mtx_lock(&softc->lock);
1227         TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1228         mtx_unlock(&softc->lock);
1229
1230         /*
1231          * Start pinging the initiator.
1232          */
1233         callout_init(&cs->cs_callout, 1);
1234         callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1235
1236         return (cs);
1237 }
1238
1239 static void
1240 cfiscsi_session_delete(struct cfiscsi_session *cs)
1241 {
1242         struct cfiscsi_softc *softc;
1243
1244         softc = &cfiscsi_softc;
1245
1246         KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1247             ("destroying session with outstanding CTL pdus"));
1248         KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1249             ("destroying session with non-empty queue"));
1250
1251         cfiscsi_session_unregister_initiator(cs);
1252         if (cs->cs_target != NULL)
1253                 cfiscsi_target_release(cs->cs_target);
1254         icl_conn_close(cs->cs_conn);
1255         icl_conn_free(cs->cs_conn);
1256
1257         mtx_lock(&softc->lock);
1258         TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1259         mtx_unlock(&softc->lock);
1260
1261         free(cs, M_CFISCSI);
1262 }
1263
1264 int
1265 cfiscsi_init(void)
1266 {
1267         struct cfiscsi_softc *softc;
1268         struct ctl_frontend *fe;
1269         int retval;
1270
1271         softc = &cfiscsi_softc;
1272         retval = 0;
1273         bzero(softc, sizeof(*softc));
1274         mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1275
1276 #ifdef ICL_KERNEL_PROXY
1277         cv_init(&softc->accept_cv, "cfiscsi_accept");
1278 #endif
1279         TAILQ_INIT(&softc->sessions);
1280         TAILQ_INIT(&softc->targets);
1281
1282         fe = &softc->fe;
1283         fe->port_type = CTL_PORT_ISCSI;
1284         /* XXX KDM what should the real number be here? */
1285         fe->num_requested_ctl_io = 4096;
1286         snprintf(softc->port_name, sizeof(softc->port_name), "iscsi");
1287         fe->port_name = softc->port_name;
1288         fe->port_online = cfiscsi_online;
1289         fe->port_offline = cfiscsi_offline;
1290         fe->onoff_arg = softc;
1291         fe->targ_enable = cfiscsi_targ_enable;
1292         fe->targ_disable = cfiscsi_targ_disable;
1293         fe->lun_enable = cfiscsi_lun_enable;
1294         fe->lun_disable = cfiscsi_lun_disable;
1295         fe->targ_lun_arg = softc;
1296         fe->ioctl = cfiscsi_ioctl;
1297         fe->devid = cfiscsi_devid;
1298         fe->fe_datamove = cfiscsi_datamove;
1299         fe->fe_done = cfiscsi_done;
1300
1301         /* XXX KDM what should we report here? */
1302         /* XXX These should probably be fetched from CTL. */
1303         fe->max_targets = 1;
1304         fe->max_target_id = 15;
1305
1306         retval = ctl_frontend_register(fe, /*master_SC*/ 1);
1307         if (retval != 0) {
1308                 CFISCSI_WARN("ctl_frontend_register() failed with error %d",
1309                     retval);
1310                 retval = 1;
1311                 goto bailout;
1312         }
1313
1314         softc->max_initiators = fe->max_initiators;
1315
1316         cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1317             sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1318             UMA_ALIGN_PTR, 0);
1319
1320         return (0);
1321
1322 bailout:
1323         return (retval);
1324 }
1325
1326 static int
1327 cfiscsi_module_event_handler(module_t mod, int what, void *arg)
1328 {
1329
1330         switch (what) {
1331         case MOD_LOAD:
1332                 return (cfiscsi_init());
1333         case MOD_UNLOAD:
1334                 return (EBUSY);
1335         default:
1336                 return (EOPNOTSUPP);
1337         }
1338 }
1339
1340 #ifdef ICL_KERNEL_PROXY
1341 static void
1342 cfiscsi_accept(struct socket *so)
1343 {
1344         struct cfiscsi_session *cs;
1345
1346         cs = cfiscsi_session_new(&cfiscsi_softc);
1347         if (cs == NULL) {
1348                 CFISCSI_WARN("failed to create session");
1349                 return;
1350         }
1351
1352         icl_conn_handoff_sock(cs->cs_conn, so);
1353         cs->cs_waiting_for_ctld = true;
1354         cv_signal(&cfiscsi_softc.accept_cv);
1355 }
1356 #endif
1357
1358 static void
1359 cfiscsi_online(void *arg)
1360 {
1361         struct cfiscsi_softc *softc;
1362
1363         softc = (struct cfiscsi_softc *)arg;
1364
1365         softc->online = 1;
1366 #ifdef ICL_KERNEL_PROXY
1367         if (softc->listener != NULL)
1368                 icl_listen_free(softc->listener);
1369         softc->listener = icl_listen_new(cfiscsi_accept);
1370 #endif
1371 }
1372
1373 static void
1374 cfiscsi_offline(void *arg)
1375 {
1376         struct cfiscsi_softc *softc;
1377         struct cfiscsi_session *cs;
1378
1379         softc = (struct cfiscsi_softc *)arg;
1380
1381         softc->online = 0;
1382
1383         mtx_lock(&softc->lock);
1384         TAILQ_FOREACH(cs, &softc->sessions, cs_next)
1385                 cfiscsi_session_terminate(cs);
1386         mtx_unlock(&softc->lock);
1387
1388 #ifdef ICL_KERNEL_PROXY
1389         icl_listen_free(softc->listener);
1390         softc->listener = NULL;
1391 #endif
1392 }
1393
1394 static int
1395 cfiscsi_targ_enable(void *arg, struct ctl_id targ_id)
1396 {
1397
1398         return (0);
1399 }
1400
1401 static int
1402 cfiscsi_targ_disable(void *arg, struct ctl_id targ_id)
1403 {
1404
1405         return (0);
1406 }
1407
1408 static void
1409 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1410 {
1411         struct cfiscsi_softc *softc;
1412         struct cfiscsi_session *cs;
1413         struct cfiscsi_target *ct;
1414         struct ctl_iscsi_handoff_params *cihp;
1415 #ifndef ICL_KERNEL_PROXY
1416         int error;
1417 #endif
1418
1419         cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1420         softc = &cfiscsi_softc;
1421
1422         CFISCSI_DEBUG("new connection from %s (%s) to %s",
1423             cihp->initiator_name, cihp->initiator_addr,
1424             cihp->target_name);
1425
1426         if (softc->online == 0) {
1427                 ci->status = CTL_ISCSI_ERROR;
1428                 snprintf(ci->error_str, sizeof(ci->error_str),
1429                     "%s: port offline", __func__);
1430                 return;
1431         }
1432
1433         ct = cfiscsi_target_find(softc, cihp->target_name);
1434         if (ct == NULL) {
1435                 ci->status = CTL_ISCSI_ERROR;
1436                 snprintf(ci->error_str, sizeof(ci->error_str),
1437                     "%s: target not found", __func__);
1438                 return;
1439         }
1440
1441 #ifdef ICL_KERNEL_PROXY
1442         mtx_lock(&cfiscsi_softc.lock);
1443         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1444                 if (cs->cs_id == cihp->socket)
1445                         break;
1446         }
1447         if (cs == NULL) {
1448                 mtx_unlock(&cfiscsi_softc.lock);
1449                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1450                 ci->status = CTL_ISCSI_ERROR;
1451                 cfiscsi_target_release(ct);
1452                 return;
1453         }
1454         mtx_unlock(&cfiscsi_softc.lock);
1455 #else
1456         cs = cfiscsi_session_new(softc);
1457         if (cs == NULL) {
1458                 ci->status = CTL_ISCSI_ERROR;
1459                 snprintf(ci->error_str, sizeof(ci->error_str),
1460                     "%s: cfiscsi_session_new failed", __func__);
1461                 cfiscsi_target_release(ct);
1462                 return;
1463         }
1464 #endif
1465         cs->cs_target = ct;
1466
1467         /*
1468          * First PDU of Full Feature phase has the same CmdSN as the last
1469          * PDU from the Login Phase received from the initiator.  Thus,
1470          * the -1 below.
1471          */
1472         cs->cs_portal_group_tag = cihp->portal_group_tag;
1473         cs->cs_cmdsn = cihp->cmdsn;
1474         cs->cs_statsn = cihp->statsn;
1475         cs->cs_max_data_segment_length = cihp->max_recv_data_segment_length;
1476         cs->cs_max_burst_length = cihp->max_burst_length;
1477         cs->cs_immediate_data = !!cihp->immediate_data;
1478         if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1479                 cs->cs_conn->ic_header_crc32c = true;
1480         if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1481                 cs->cs_conn->ic_data_crc32c = true;
1482
1483         strlcpy(cs->cs_initiator_name,
1484             cihp->initiator_name, sizeof(cs->cs_initiator_name));
1485         strlcpy(cs->cs_initiator_addr,
1486             cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1487         strlcpy(cs->cs_initiator_alias,
1488             cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1489
1490 #ifdef ICL_KERNEL_PROXY
1491         cs->cs_login_phase = false;
1492 #else
1493         error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1494         if (error != 0) {
1495                 cfiscsi_session_delete(cs);
1496                 ci->status = CTL_ISCSI_ERROR;
1497                 snprintf(ci->error_str, sizeof(ci->error_str),
1498                     "%s: icl_conn_handoff failed with error %d",
1499                     __func__, error);
1500                 return;
1501         }
1502 #endif
1503
1504         /*
1505          * Register initiator with CTL.
1506          */
1507         cfiscsi_session_register_initiator(cs);
1508
1509 #ifdef ICL_KERNEL_PROXY
1510         /*
1511          * First PDU of the Full Feature phase has likely already arrived.
1512          * We have to pick it up and execute properly.
1513          */
1514         if (cs->cs_login_pdu != NULL) {
1515                 CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1516                 cfiscsi_pdu_handle(cs->cs_login_pdu);
1517                 cs->cs_login_pdu = NULL;
1518         }
1519 #endif
1520
1521         ci->status = CTL_ISCSI_OK;
1522 }
1523
1524 static void
1525 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1526 {
1527         struct ctl_iscsi_list_params *cilp;
1528         struct cfiscsi_session *cs;
1529         struct cfiscsi_softc *softc;
1530         struct sbuf *sb;
1531         int error;
1532
1533         cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1534         softc = &cfiscsi_softc;
1535
1536         sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1537         if (sb == NULL) {
1538                 ci->status = CTL_ISCSI_ERROR;
1539                 snprintf(ci->error_str, sizeof(ci->error_str),
1540                     "Unable to allocate %d bytes for iSCSI session list",
1541                     cilp->alloc_len);
1542                 return;
1543         }
1544
1545         sbuf_printf(sb, "<ctlislist>\n");
1546         mtx_lock(&softc->lock);
1547         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1548 #ifdef ICL_KERNEL_PROXY
1549                 if (cs->cs_target == NULL)
1550                         continue;
1551 #endif
1552                 error = sbuf_printf(sb, "<connection id=\"%d\">"
1553                     "<initiator>%s</initiator>"
1554                     "<initiator_addr>%s</initiator_addr>"
1555                     "<initiator_alias>%s</initiator_alias>"
1556                     "<target>%s</target>"
1557                     "<target_alias>%s</target_alias>"
1558                     "<header_digest>%s</header_digest>"
1559                     "<data_digest>%s</data_digest>"
1560                     "<max_data_segment_length>%zd</max_data_segment_length>"
1561                     "<immediate_data>%d</immediate_data>"
1562                     "<iser>%d</iser>"
1563                     "</connection>\n",
1564                     cs->cs_id,
1565                     cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1566                     cs->cs_target->ct_name, cs->cs_target->ct_alias,
1567                     cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1568                     cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1569                     cs->cs_max_data_segment_length,
1570                     cs->cs_immediate_data,
1571                     cs->cs_conn->ic_iser);
1572                 if (error != 0)
1573                         break;
1574         }
1575         mtx_unlock(&softc->lock);
1576         error = sbuf_printf(sb, "</ctlislist>\n");
1577         if (error != 0) {
1578                 sbuf_delete(sb);
1579                 ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1580                 snprintf(ci->error_str, sizeof(ci->error_str),
1581                     "Out of space, %d bytes is too small", cilp->alloc_len);
1582                 return;
1583         }
1584         sbuf_finish(sb);
1585
1586         error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1587         cilp->fill_len = sbuf_len(sb) + 1;
1588         ci->status = CTL_ISCSI_OK;
1589         sbuf_delete(sb);
1590 }
1591
1592 static void
1593 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1594 {
1595         struct icl_pdu *response;
1596         struct iscsi_bhs_asynchronous_message *bhsam;
1597         struct ctl_iscsi_terminate_params *citp;
1598         struct cfiscsi_session *cs;
1599         struct cfiscsi_softc *softc;
1600         int found = 0;
1601
1602         citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1603         softc = &cfiscsi_softc;
1604
1605         mtx_lock(&softc->lock);
1606         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1607                 if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1608                     strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1609                     strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1610                         continue;
1611
1612                 response = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
1613                 if (response == NULL) {
1614                         /*
1615                          * Oh well.  Just terminate the connection.
1616                          */
1617                 } else {
1618                         bhsam = (struct iscsi_bhs_asynchronous_message *)
1619                             response->ip_bhs;
1620                         bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1621                         bhsam->bhsam_flags = 0x80;
1622                         bhsam->bhsam_0xffffffff = 0xffffffff;
1623                         bhsam->bhsam_async_event =
1624                             BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1625                         cfiscsi_pdu_queue(response);
1626                 }
1627                 cfiscsi_session_terminate(cs);
1628                 found++;
1629         }
1630         mtx_unlock(&softc->lock);
1631
1632         if (found == 0) {
1633                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1634                 snprintf(ci->error_str, sizeof(ci->error_str),
1635                     "No matching connections found");
1636                 return;
1637         }
1638
1639         ci->status = CTL_ISCSI_OK;
1640 }
1641
1642 static void
1643 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1644 {
1645         struct icl_pdu *response;
1646         struct iscsi_bhs_asynchronous_message *bhsam;
1647         struct ctl_iscsi_logout_params *cilp;
1648         struct cfiscsi_session *cs;
1649         struct cfiscsi_softc *softc;
1650         int found = 0;
1651
1652         cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1653         softc = &cfiscsi_softc;
1654
1655         mtx_lock(&softc->lock);
1656         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1657                 if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1658                     strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1659                     strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1660                         continue;
1661
1662                 response = icl_pdu_new_bhs(cs->cs_conn, M_NOWAIT);
1663                 if (response == NULL) {
1664                         ci->status = CTL_ISCSI_ERROR;
1665                         snprintf(ci->error_str, sizeof(ci->error_str),
1666                             "Unable to allocate memory");
1667                         mtx_unlock(&softc->lock);
1668                         return;
1669                 }
1670                 bhsam =
1671                     (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1672                 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1673                 bhsam->bhsam_flags = 0x80;
1674                 bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1675                 bhsam->bhsam_parameter3 = htons(10);
1676                 cfiscsi_pdu_queue(response);
1677                 found++;
1678         }
1679         mtx_unlock(&softc->lock);
1680
1681         if (found == 0) {
1682                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1683                 snprintf(ci->error_str, sizeof(ci->error_str),
1684                     "No matching connections found");
1685                 return;
1686         }
1687
1688         ci->status = CTL_ISCSI_OK;
1689 }
1690
1691 #ifdef ICL_KERNEL_PROXY
1692 static void
1693 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1694 {
1695         struct ctl_iscsi_listen_params *cilp;
1696         struct sockaddr *sa;
1697         int error;
1698
1699         cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1700
1701         if (cfiscsi_softc.listener == NULL) {
1702                 CFISCSI_DEBUG("no listener");
1703                 snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1704                 ci->status = CTL_ISCSI_ERROR;
1705                 return;
1706         }
1707
1708         error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1709         if (error != 0) {
1710                 CFISCSI_DEBUG("getsockaddr, error %d", error);
1711                 snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1712                 ci->status = CTL_ISCSI_ERROR;
1713                 return;
1714         }
1715
1716         error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1717             cilp->socktype, cilp->protocol, sa);
1718         if (error != 0) {
1719                 free(sa, M_SONAME);
1720                 CFISCSI_DEBUG("icl_listen_add, error %d", error);
1721                 snprintf(ci->error_str, sizeof(ci->error_str),
1722                     "icl_listen_add failed, error %d", error);
1723                 ci->status = CTL_ISCSI_ERROR;
1724                 return;
1725         }
1726
1727         ci->status = CTL_ISCSI_OK;
1728 }
1729
1730 static void
1731 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1732 {
1733         struct ctl_iscsi_accept_params *ciap;
1734         struct cfiscsi_session *cs;
1735         int error;
1736
1737         ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1738
1739         mtx_lock(&cfiscsi_softc.lock);
1740         for (;;) {
1741                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1742                         if (cs->cs_waiting_for_ctld)
1743                                 break;
1744                 }
1745                 if (cs != NULL)
1746                         break;
1747                 error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1748                 if (error != 0) {
1749                         mtx_unlock(&cfiscsi_softc.lock);
1750                         snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1751                         ci->status = CTL_ISCSI_ERROR;
1752                         return;
1753                 }
1754         }
1755         mtx_unlock(&cfiscsi_softc.lock);
1756
1757         cs->cs_waiting_for_ctld = false;
1758         cs->cs_login_phase = true;
1759
1760         ciap->connection_id = cs->cs_id;
1761         ci->status = CTL_ISCSI_OK;
1762 }
1763
1764 static void
1765 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
1766 {
1767         struct ctl_iscsi_send_params *cisp;
1768         struct cfiscsi_session *cs;
1769         struct icl_pdu *ip;
1770         size_t datalen;
1771         void *data;
1772         int error;
1773
1774         cisp = (struct ctl_iscsi_send_params *)&(ci->data);
1775
1776         mtx_lock(&cfiscsi_softc.lock);
1777         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1778                 if (cs->cs_id == cisp->connection_id)
1779                         break;
1780         }
1781         if (cs == NULL) {
1782                 mtx_unlock(&cfiscsi_softc.lock);
1783                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1784                 ci->status = CTL_ISCSI_ERROR;
1785                 return;
1786         }
1787         mtx_unlock(&cfiscsi_softc.lock);
1788
1789 #if 0
1790         if (cs->cs_login_phase == false)
1791                 return (EBUSY);
1792 #endif
1793
1794         if (cs->cs_terminating) {
1795                 snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
1796                 ci->status = CTL_ISCSI_ERROR;
1797                 return;
1798         }
1799
1800         datalen = cisp->data_segment_len;
1801         /*
1802          * XXX
1803          */
1804         //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
1805         if (datalen > 65535) {
1806                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1807                 ci->status = CTL_ISCSI_ERROR;
1808                 return;
1809         }
1810         if (datalen > 0) {
1811                 data = malloc(datalen, M_CFISCSI, M_WAITOK);
1812                 error = copyin(cisp->data_segment, data, datalen);
1813                 if (error != 0) {
1814                         free(data, M_CFISCSI);
1815                         snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
1816                         ci->status = CTL_ISCSI_ERROR;
1817                         return;
1818                 }
1819         }
1820
1821         ip = icl_pdu_new_bhs(cs->cs_conn, M_WAITOK);
1822         memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
1823         if (datalen > 0) {
1824                 icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1825                 free(data, M_CFISCSI);
1826         }
1827         icl_pdu_queue(ip);
1828         ci->status = CTL_ISCSI_OK;
1829 }
1830
1831 static void
1832 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
1833 {
1834         struct ctl_iscsi_receive_params *cirp;
1835         struct cfiscsi_session *cs;
1836         struct icl_pdu *ip;
1837         void *data;
1838
1839         cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
1840
1841         mtx_lock(&cfiscsi_softc.lock);
1842         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1843                 if (cs->cs_id == cirp->connection_id)
1844                         break;
1845         }
1846         if (cs == NULL) {
1847                 mtx_unlock(&cfiscsi_softc.lock);
1848                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
1849                 ci->status = CTL_ISCSI_ERROR;
1850                 return;
1851         }
1852         mtx_unlock(&cfiscsi_softc.lock);
1853
1854 #if 0
1855         if (is->is_login_phase == false)
1856                 return (EBUSY);
1857 #endif
1858
1859         CFISCSI_SESSION_LOCK(cs);
1860         while (cs->cs_login_pdu == NULL &&
1861             cs->cs_terminating == false)
1862                 cv_wait(&cs->cs_login_cv, &cs->cs_lock);
1863         if (cs->cs_terminating) {
1864                 CFISCSI_SESSION_UNLOCK(cs);
1865                 snprintf(ci->error_str, sizeof(ci->error_str), "connection terminating");
1866                 ci->status = CTL_ISCSI_ERROR;
1867                 return;
1868         }
1869         ip = cs->cs_login_pdu;
1870         cs->cs_login_pdu = NULL;
1871         CFISCSI_SESSION_UNLOCK(cs);
1872
1873         if (ip->ip_data_len > cirp->data_segment_len) {
1874                 icl_pdu_free(ip);
1875                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
1876                 ci->status = CTL_ISCSI_ERROR;
1877                 return;
1878         }
1879
1880         copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
1881         if (ip->ip_data_len > 0) {
1882                 data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
1883                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1884                 copyout(data, cirp->data_segment, ip->ip_data_len);
1885                 free(data, M_CFISCSI);
1886         }
1887
1888         icl_pdu_free(ip);
1889         ci->status = CTL_ISCSI_OK;
1890 }
1891
1892 static void
1893 cfiscsi_ioctl_close(struct ctl_iscsi *ci)
1894 {
1895         /*
1896          * XXX
1897          */
1898 }
1899 #endif /* !ICL_KERNEL_PROXY */
1900
1901 static int
1902 cfiscsi_ioctl(struct cdev *dev,
1903     u_long cmd, caddr_t addr, int flag, struct thread *td)
1904 {
1905         struct ctl_iscsi *ci;
1906
1907         if (cmd != CTL_ISCSI)
1908                 return (ENOTTY);
1909
1910         ci = (struct ctl_iscsi *)addr;
1911         switch (ci->type) {
1912         case CTL_ISCSI_HANDOFF:
1913                 cfiscsi_ioctl_handoff(ci);
1914                 break;
1915         case CTL_ISCSI_LIST:
1916                 cfiscsi_ioctl_list(ci);
1917                 break;
1918         case CTL_ISCSI_TERMINATE:
1919                 cfiscsi_ioctl_terminate(ci);
1920                 break;
1921         case CTL_ISCSI_LOGOUT:
1922                 cfiscsi_ioctl_logout(ci);
1923                 break;
1924 #ifdef ICL_KERNEL_PROXY
1925         case CTL_ISCSI_LISTEN:
1926                 cfiscsi_ioctl_listen(ci);
1927                 break;
1928         case CTL_ISCSI_ACCEPT:
1929                 cfiscsi_ioctl_accept(ci);
1930                 break;
1931         case CTL_ISCSI_SEND:
1932                 cfiscsi_ioctl_send(ci);
1933                 break;
1934         case CTL_ISCSI_RECEIVE:
1935                 cfiscsi_ioctl_receive(ci);
1936                 break;
1937         case CTL_ISCSI_CLOSE:
1938                 cfiscsi_ioctl_close(ci);
1939                 break;
1940 #endif /* ICL_KERNEL_PROXY */
1941         default:
1942                 ci->status = CTL_ISCSI_ERROR;
1943                 snprintf(ci->error_str, sizeof(ci->error_str),
1944                     "%s: invalid iSCSI request type %d", __func__, ci->type);
1945                 break;
1946         }
1947
1948         return (0);
1949 }
1950
1951 static int
1952 cfiscsi_devid(struct ctl_scsiio *ctsio, int alloc_len)
1953 {
1954         struct cfiscsi_session *cs;
1955         struct scsi_vpd_device_id *devid_ptr;
1956         struct scsi_vpd_id_descriptor *desc, *desc1;
1957         struct scsi_vpd_id_descriptor *desc2, *desc3; /* for types 4h and 5h */
1958         struct scsi_vpd_id_t10 *t10id;
1959         struct ctl_lun *lun;
1960         const struct icl_pdu *request;
1961         size_t devid_len, wwpn_len;
1962
1963         lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
1964         request = ctsio->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1965         cs = PDU_SESSION(request);
1966
1967         wwpn_len = strlen(cs->cs_target->ct_name);
1968         wwpn_len += strlen(",t,0x01");
1969         wwpn_len += 1; /* '\0' */
1970         if ((wwpn_len % 4) != 0)
1971                 wwpn_len += (4 - (wwpn_len % 4));
1972
1973         devid_len = sizeof(struct scsi_vpd_device_id) +
1974                 sizeof(struct scsi_vpd_id_descriptor) +
1975                 sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN +
1976                 sizeof(struct scsi_vpd_id_descriptor) + wwpn_len +
1977                 sizeof(struct scsi_vpd_id_descriptor) +
1978                 sizeof(struct scsi_vpd_id_rel_trgt_port_id) +
1979                 sizeof(struct scsi_vpd_id_descriptor) +
1980                 sizeof(struct scsi_vpd_id_trgt_port_grp_id);
1981
1982         ctsio->kern_data_ptr = malloc(devid_len, M_CTL, M_WAITOK | M_ZERO);
1983         devid_ptr = (struct scsi_vpd_device_id *)ctsio->kern_data_ptr;
1984         ctsio->kern_sg_entries = 0;
1985
1986         if (devid_len < alloc_len) {
1987                 ctsio->residual = alloc_len - devid_len;
1988                 ctsio->kern_data_len = devid_len;
1989                 ctsio->kern_total_len = devid_len;
1990         } else {
1991                 ctsio->residual = 0;
1992                 ctsio->kern_data_len = alloc_len;
1993                 ctsio->kern_total_len = alloc_len;
1994         }
1995         ctsio->kern_data_resid = 0;
1996         ctsio->kern_rel_offset = 0;
1997         ctsio->kern_sg_entries = 0;
1998
1999         desc = (struct scsi_vpd_id_descriptor *)devid_ptr->desc_list;
2000         t10id = (struct scsi_vpd_id_t10 *)&desc->identifier[0];
2001         desc1 = (struct scsi_vpd_id_descriptor *)(&desc->identifier[0] +
2002             sizeof(struct scsi_vpd_id_t10) + CTL_DEVID_LEN);
2003         desc2 = (struct scsi_vpd_id_descriptor *)(&desc1->identifier[0] +
2004             wwpn_len);
2005         desc3 = (struct scsi_vpd_id_descriptor *)(&desc2->identifier[0] +
2006             sizeof(struct scsi_vpd_id_rel_trgt_port_id));
2007
2008         if (lun != NULL)
2009                 devid_ptr->device = (SID_QUAL_LU_CONNECTED << 5) |
2010                     lun->be_lun->lun_type;
2011         else
2012                 devid_ptr->device = (SID_QUAL_LU_OFFLINE << 5) | T_DIRECT;
2013
2014         devid_ptr->page_code = SVPD_DEVICE_ID;
2015
2016         scsi_ulto2b(devid_len - 4, devid_ptr->length);
2017
2018         /*
2019          * We're using a LUN association here.  i.e., this device ID is a
2020          * per-LUN identifier.
2021          */
2022         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_ASCII;
2023         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_LUN | SVPD_ID_TYPE_T10;
2024         desc->length = sizeof(*t10id) + CTL_DEVID_LEN;
2025         strncpy((char *)t10id->vendor, CTL_VENDOR, sizeof(t10id->vendor));
2026
2027         /*
2028          * If we've actually got a backend, copy the device id from the
2029          * per-LUN data.  Otherwise, set it to all spaces.
2030          */
2031         if (lun != NULL) {
2032                 /*
2033                  * Copy the backend's LUN ID.
2034                  */
2035                 strncpy((char *)t10id->vendor_spec_id,
2036                     (char *)lun->be_lun->device_id, CTL_DEVID_LEN);
2037         } else {
2038                 /*
2039                  * No backend, set this to spaces.
2040                  */
2041                 memset(t10id->vendor_spec_id, 0x20, CTL_DEVID_LEN);
2042         }
2043
2044         /*
2045          * desc1 is for the WWPN which is a port asscociation.
2046          */
2047         desc1->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2048         desc1->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2049             SVPD_ID_TYPE_SCSI_NAME;
2050         desc1->length = wwpn_len;
2051         snprintf(desc1->identifier, wwpn_len, "%s,t,0x%x",
2052             cs->cs_target->ct_name, cs->cs_portal_group_tag);
2053
2054         /*
2055          * desc2 is for the Relative Target Port(type 4h) identifier
2056          */
2057         desc2->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_BINARY;
2058         desc2->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2059             SVPD_ID_TYPE_RELTARG;
2060         desc2->length = 4;
2061         desc2->identifier[3] = 1;
2062
2063         /*
2064          * desc3 is for the Target Port Group(type 5h) identifier
2065          */
2066         desc3->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_BINARY;
2067         desc3->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2068             SVPD_ID_TYPE_TPORTGRP;
2069         desc3->length = 4;
2070         desc3->identifier[3] = 1;
2071
2072         ctsio->scsi_status = SCSI_STATUS_OK;
2073
2074         ctsio->be_move_done = ctl_config_move_done;
2075         ctl_datamove((union ctl_io *)ctsio);
2076
2077         return (CTL_RETVAL_COMPLETE);
2078 }
2079
2080 static void
2081 cfiscsi_target_hold(struct cfiscsi_target *ct)
2082 {
2083
2084         refcount_acquire(&ct->ct_refcount);
2085 }
2086
2087 static void
2088 cfiscsi_target_release(struct cfiscsi_target *ct)
2089 {
2090         struct cfiscsi_softc *softc;
2091
2092         softc = ct->ct_softc;
2093         mtx_lock(&softc->lock);
2094         if (refcount_release(&ct->ct_refcount)) {
2095                 TAILQ_REMOVE(&softc->targets, ct, ct_next);
2096                 mtx_unlock(&softc->lock);
2097                 free(ct, M_CFISCSI);
2098
2099                 return;
2100         }
2101         mtx_unlock(&softc->lock);
2102 }
2103
2104 static struct cfiscsi_target *
2105 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name)
2106 {
2107         struct cfiscsi_target *ct;
2108
2109         mtx_lock(&softc->lock);
2110         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2111                 if (strcmp(name, ct->ct_name) != 0)
2112                         continue;
2113                 cfiscsi_target_hold(ct);
2114                 mtx_unlock(&softc->lock);
2115                 return (ct);
2116         }
2117         mtx_unlock(&softc->lock);
2118
2119         return (NULL);
2120 }
2121
2122 static struct cfiscsi_target *
2123 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2124     const char *alias)
2125 {
2126         struct cfiscsi_target *ct, *newct;
2127         int i;
2128
2129         if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2130                 return (NULL);
2131
2132         newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2133
2134         mtx_lock(&softc->lock);
2135         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2136                 if (strcmp(name, ct->ct_name) != 0)
2137                         continue;
2138                 cfiscsi_target_hold(ct);
2139                 mtx_unlock(&softc->lock);
2140                 free(newct, M_CFISCSI);
2141                 return (ct);
2142         }
2143
2144         for (i = 0; i < CTL_MAX_LUNS; i++)
2145                 newct->ct_luns[i] = -1;
2146
2147         strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2148         if (alias != NULL)
2149                 strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2150         refcount_init(&newct->ct_refcount, 1);
2151         newct->ct_softc = softc;
2152         TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2153         mtx_unlock(&softc->lock);
2154
2155         return (newct);
2156 }
2157
2158 /*
2159  * Takes LUN from the target space and returns LUN from the CTL space.
2160  */
2161 static uint32_t
2162 cfiscsi_map_lun(void *arg, uint32_t lun)
2163 {
2164         struct cfiscsi_session *cs;
2165
2166         cs = arg;
2167
2168         if (lun >= CTL_MAX_LUNS) {
2169                 CFISCSI_DEBUG("requested lun number %d is higher "
2170                     "than maximum %d", lun, CTL_MAX_LUNS - 1);
2171                 return (0xffffffff);
2172         }
2173
2174         if (cs->cs_target->ct_luns[lun] < 0)
2175                 return (0xffffffff);
2176
2177         return (cs->cs_target->ct_luns[lun]);
2178 }
2179
2180 static int
2181 cfiscsi_target_set_lun(struct cfiscsi_target *ct,
2182     unsigned long lun_id, unsigned long ctl_lun_id)
2183 {
2184
2185         if (lun_id >= CTL_MAX_LUNS) {
2186                 CFISCSI_WARN("requested lun number %ld is higher "
2187                     "than maximum %d", lun_id, CTL_MAX_LUNS - 1);
2188                 return (-1);
2189         }
2190
2191         if (ct->ct_luns[lun_id] >= 0) {
2192                 /*
2193                  * CTL calls cfiscsi_lun_enable() twice for each LUN - once
2194                  * when the LUN is created, and a second time just before
2195                  * the port is brought online; don't emit warnings
2196                  * for that case.
2197                  */
2198                 if (ct->ct_luns[lun_id] == ctl_lun_id)
2199                         return (0);
2200                 CFISCSI_WARN("lun %ld already allocated", lun_id);
2201                 return (-1);
2202         }
2203
2204 #if 0
2205         CFISCSI_DEBUG("adding mapping for lun %ld, target %s "
2206             "to ctl lun %ld", lun_id, ct->ct_name, ctl_lun_id);
2207 #endif
2208
2209         ct->ct_luns[lun_id] = ctl_lun_id;
2210         cfiscsi_target_hold(ct);
2211
2212         return (0);
2213 }
2214
2215 static int
2216 cfiscsi_target_unset_lun(struct cfiscsi_target *ct, unsigned long lun_id)
2217 {
2218
2219         if (ct->ct_luns[lun_id] < 0) {
2220                 CFISCSI_WARN("lun %ld not allocated", lun_id);
2221                 return (-1);
2222         }
2223
2224         ct->ct_luns[lun_id] = -1;
2225         cfiscsi_target_release(ct);
2226
2227         return (0);
2228 }
2229
2230 static int
2231 cfiscsi_lun_enable(void *arg, struct ctl_id target_id, int lun_id)
2232 {
2233         struct cfiscsi_softc *softc;
2234         struct cfiscsi_target *ct;
2235         struct ctl_be_lun_option *opt;
2236         const char *target = NULL, *target_alias = NULL;
2237         const char *lun = NULL;
2238         unsigned long tmp;
2239
2240         softc = (struct cfiscsi_softc *)arg;
2241
2242         STAILQ_FOREACH(opt,
2243             &control_softc->ctl_luns[lun_id]->be_lun->options, links) {
2244                 if (strcmp(opt->name, "cfiscsi_target") == 0)
2245                         target = opt->value;
2246                 else if (strcmp(opt->name, "cfiscsi_target_alias") == 0)
2247                         target_alias = opt->value;
2248                 else if (strcmp(opt->name, "cfiscsi_lun") == 0)
2249                         lun = opt->value;
2250         }
2251
2252         if (target == NULL && lun == NULL)
2253                 return (0);
2254
2255         if (target == NULL || lun == NULL) {
2256                 CFISCSI_WARN("lun added with cfiscsi_target, but without "
2257                     "cfiscsi_lun, or the other way around; ignoring");
2258                 return (0);
2259         }
2260
2261         ct = cfiscsi_target_find_or_create(softc, target, target_alias);
2262         if (ct == NULL) {
2263                 CFISCSI_WARN("failed to create target \"%s\"", target);
2264                 return (0);
2265         }
2266
2267         tmp = strtoul(lun, NULL, 10);
2268         cfiscsi_target_set_lun(ct, tmp, lun_id);
2269         cfiscsi_target_release(ct);
2270         return (0);
2271 }
2272
2273 static int
2274 cfiscsi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
2275 {
2276         struct cfiscsi_softc *softc;
2277         struct cfiscsi_target *ct;
2278         int i;
2279
2280         softc = (struct cfiscsi_softc *)arg;
2281
2282         mtx_lock(&softc->lock);
2283         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2284                 for (i = 0; i < CTL_MAX_LUNS; i++) {
2285                         if (ct->ct_luns[i] < 0)
2286                                 continue;
2287                         if (ct->ct_luns[i] != lun_id)
2288                                 continue;
2289                         mtx_unlock(&softc->lock);
2290                         cfiscsi_target_unset_lun(ct, i);
2291                         return (0);
2292                 }
2293         }
2294         mtx_unlock(&softc->lock);
2295         return (0);
2296 }
2297
2298 static void
2299 cfiscsi_datamove_in(union ctl_io *io)
2300 {
2301         struct cfiscsi_session *cs;
2302         struct icl_pdu *request, *response;
2303         const struct iscsi_bhs_scsi_command *bhssc;
2304         struct iscsi_bhs_data_in *bhsdi;
2305         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2306         size_t copy_len, len, off;
2307         const char *addr;
2308         int ctl_sg_count, error, i;
2309
2310         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2311         cs = PDU_SESSION(request);
2312
2313         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2314         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2315             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2316             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2317
2318         if (io->scsiio.kern_sg_entries > 0) {
2319                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2320                 ctl_sg_count = io->scsiio.kern_sg_entries;
2321         } else {
2322                 ctl_sglist = &ctl_sg_entry;
2323                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2324                 ctl_sglist->len = io->scsiio.kern_data_len;
2325                 ctl_sg_count = 1;
2326         }
2327
2328         /*
2329          * We need to record it so that we can properly report
2330          * underflow/underflow.
2331          */
2332         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2333
2334         /*
2335          * This is the offset within the current SCSI command;
2336          * i.e. for the first call of datamove(), it will be 0,
2337          * and for subsequent ones it will be the sum of lengths
2338          * of previous ones.
2339          */
2340         off = htonl(io->scsiio.kern_rel_offset);
2341
2342         i = 0;
2343         addr = NULL;
2344         len = 0;
2345         response = NULL;
2346         bhsdi = NULL;
2347         for (;;) {
2348                 KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2349                 if (response == NULL) {
2350                         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2351                         if (response == NULL) {
2352                                 CFISCSI_SESSION_WARN(cs, "failed to "
2353                                     "allocate memory; dropping connection");
2354                                 ctl_set_busy(&io->scsiio);
2355                                 io->scsiio.be_move_done(io);
2356                                 cfiscsi_session_terminate(cs);
2357                                 return;
2358                         }
2359                         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2360                         bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2361                         bhsdi->bhsdi_initiator_task_tag =
2362                             bhssc->bhssc_initiator_task_tag;
2363                         bhsdi->bhsdi_datasn = htonl(PDU_EXPDATASN(request));
2364                         PDU_EXPDATASN(request)++;
2365                         bhsdi->bhsdi_buffer_offset = htonl(off);
2366                 }
2367
2368                 if (len == 0) {
2369                         addr = ctl_sglist[i].addr;
2370                         len = ctl_sglist[i].len;
2371                         KASSERT(len > 0, ("len <= 0"));
2372                 }
2373
2374                 copy_len = len;
2375                 if (response->ip_data_len + copy_len >
2376                     cs->cs_max_data_segment_length)
2377                         copy_len = cs->cs_max_data_segment_length -
2378                             response->ip_data_len;
2379                 KASSERT(copy_len <= len, ("copy_len > len"));
2380                 error = icl_pdu_append_data(response, addr, copy_len, M_NOWAIT);
2381                 if (error != 0) {
2382                         CFISCSI_SESSION_WARN(cs, "failed to "
2383                             "allocate memory; dropping connection");
2384                         icl_pdu_free(response);
2385                         ctl_set_busy(&io->scsiio);
2386                         io->scsiio.be_move_done(io);
2387                         cfiscsi_session_terminate(cs);
2388                         return;
2389                 }
2390                 addr += copy_len;
2391                 len -= copy_len;
2392                 off += copy_len;
2393                 io->scsiio.ext_data_filled += copy_len;
2394
2395                 if (len == 0) {
2396                         /*
2397                          * End of scatter-gather segment;
2398                          * proceed to the next one...
2399                          */
2400                         if (i == ctl_sg_count - 1) {
2401                                 /*
2402                                  * ... unless this was the last one.
2403                                  */
2404                                 break;
2405                         }
2406                         i++;
2407                 }
2408
2409                 if (response->ip_data_len == cs->cs_max_data_segment_length) {
2410                         /*
2411                          * Can't stuff more data into the current PDU;
2412                          * queue it.  Note that's not enough to check
2413                          * for kern_data_resid == 0 instead; there
2414                          * may be several Data-In PDUs for the final
2415                          * call to cfiscsi_datamove(), and we want
2416                          * to set the F flag only on the last of them.
2417                          */
2418                         if (off == io->scsiio.kern_total_len)
2419                                 bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2420                         KASSERT(response->ip_data_len > 0,
2421                             ("sending empty Data-In"));
2422                         cfiscsi_pdu_queue(response);
2423                         response = NULL;
2424                         bhsdi = NULL;
2425                 }
2426         }
2427         KASSERT(i == ctl_sg_count - 1, ("missed SG segment"));
2428         KASSERT(len == 0, ("missed data from SG segment"));
2429         if (response != NULL) {
2430                 if (off == io->scsiio.kern_total_len) {
2431                         bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2432 #if 0
2433                 } else {
2434                         CFISCSI_SESSION_DEBUG(cs, "not setting the F flag; "
2435                             "have %zd, need %zd", off,
2436                             (size_t)io->scsiio.kern_total_len);
2437 #endif
2438                 }
2439                 KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2440                 cfiscsi_pdu_queue(response);
2441         }
2442
2443         io->scsiio.be_move_done(io);
2444 }
2445
2446 static void
2447 cfiscsi_datamove_out(union ctl_io *io)
2448 {
2449         struct cfiscsi_session *cs;
2450         struct icl_pdu *request, *response;
2451         const struct iscsi_bhs_scsi_command *bhssc;
2452         struct iscsi_bhs_r2t *bhsr2t;
2453         struct cfiscsi_data_wait *cdw;
2454         uint32_t target_transfer_tag;
2455         bool done;
2456
2457         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2458         cs = PDU_SESSION(request);
2459
2460         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2461         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2462             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2463             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2464
2465         /*
2466          * We need to record it so that we can properly report
2467          * underflow/underflow.
2468          */
2469         PDU_TOTAL_TRANSFER_LEN(request) = io->scsiio.kern_total_len;
2470
2471         CFISCSI_SESSION_LOCK(cs);
2472         target_transfer_tag = cs->cs_target_transfer_tag;
2473         cs->cs_target_transfer_tag++;
2474         CFISCSI_SESSION_UNLOCK(cs);
2475
2476 #if 0
2477         CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2478             "task tag 0x%x, target transfer tag 0x%x",
2479             bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2480 #endif
2481         cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
2482         if (cdw == NULL) {
2483                 CFISCSI_SESSION_WARN(cs, "failed to "
2484                     "allocate memory; dropping connection");
2485                 ctl_set_busy(&io->scsiio);
2486                 io->scsiio.be_move_done(io);
2487                 cfiscsi_session_terminate(cs);
2488                 return;
2489         }
2490         cdw->cdw_ctl_io = io;
2491         cdw->cdw_target_transfer_tag = htonl(target_transfer_tag);
2492         cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2493
2494         if (cs->cs_immediate_data && icl_pdu_data_segment_length(request) > 0) {
2495                 done = cfiscsi_handle_data_segment(request, cdw);
2496                 if (done) {
2497                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2498                         io->scsiio.be_move_done(io);
2499                         return;
2500                 }
2501
2502 #if 0
2503                 if (io->scsiio.ext_data_filled != 0)
2504                         CFISCSI_SESSION_DEBUG(cs, "got %zd bytes of immediate data, need %zd",
2505                             io->scsiio.ext_data_filled, io->scsiio.kern_data_len);
2506 #endif
2507         }
2508
2509         CFISCSI_SESSION_LOCK(cs);
2510         TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2511         CFISCSI_SESSION_UNLOCK(cs);
2512
2513         /*
2514          * XXX: We should limit the number of outstanding R2T PDUs
2515          *      per task to MaxOutstandingR2T.
2516          */
2517         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2518         if (response == NULL) {
2519                 CFISCSI_SESSION_WARN(cs, "failed to "
2520                     "allocate memory; dropping connection");
2521                 ctl_set_busy(&io->scsiio);
2522                 io->scsiio.be_move_done(io);
2523                 cfiscsi_session_terminate(cs);
2524                 return;
2525         }
2526         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2527         bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2528         bhsr2t->bhsr2t_flags = 0x80;
2529         bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2530         bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2531         bhsr2t->bhsr2t_target_transfer_tag = htonl(target_transfer_tag);
2532         /*
2533          * XXX: Here we assume that cfiscsi_datamove() won't ever
2534          *      be running concurrently on several CPUs for a given
2535          *      command.
2536          */
2537         bhsr2t->bhsr2t_r2tsn = htonl(PDU_R2TSN(request));
2538         PDU_R2TSN(request)++;
2539         /*
2540          * This is the offset within the current SCSI command;
2541          * i.e. for the first call of datamove(), it will be 0,
2542          * and for subsequent ones it will be the sum of lengths
2543          * of previous ones.
2544          *
2545          * The ext_data_filled is to account for unsolicited
2546          * (immediate) data that might have already arrived.
2547          */
2548         bhsr2t->bhsr2t_buffer_offset =
2549             htonl(io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled);
2550         /*
2551          * This is the total length (sum of S/G lengths) this call
2552          * to cfiscsi_datamove() is supposed to handle.
2553          *
2554          * XXX: Limit it to MaxBurstLength.
2555          */
2556         bhsr2t->bhsr2t_desired_data_transfer_length =
2557             htonl(io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
2558         cfiscsi_pdu_queue(response);
2559 }
2560
2561 static void
2562 cfiscsi_datamove(union ctl_io *io)
2563 {
2564
2565         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2566                 cfiscsi_datamove_in(io);
2567         else
2568                 cfiscsi_datamove_out(io);
2569 }
2570
2571 static void
2572 cfiscsi_scsi_command_done(union ctl_io *io)
2573 {
2574         struct icl_pdu *request, *response;
2575         struct iscsi_bhs_scsi_command *bhssc;
2576         struct iscsi_bhs_scsi_response *bhssr;
2577 #ifdef DIAGNOSTIC
2578         struct cfiscsi_data_wait *cdw;
2579 #endif
2580         struct cfiscsi_session *cs;
2581         uint16_t sense_length;
2582
2583         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2584         cs = PDU_SESSION(request);
2585         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2586         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2587             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2588             ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2589
2590         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2591         //    bhssc->bhssc_initiator_task_tag);
2592
2593 #ifdef DIAGNOSTIC
2594         CFISCSI_SESSION_LOCK(cs);
2595         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2596                 KASSERT(bhssc->bhssc_initiator_task_tag !=
2597                     cdw->cdw_initiator_task_tag, ("dangling cdw"));
2598         CFISCSI_SESSION_UNLOCK(cs);
2599 #endif
2600
2601         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2602         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2603         bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2604         bhssr->bhssr_flags = 0x80;
2605         /*
2606          * XXX: We don't deal with bidirectional under/overflows;
2607          *      does anything actually support those?
2608          */
2609         if (PDU_TOTAL_TRANSFER_LEN(request) <
2610             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2611                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2612                 bhssr->bhssr_residual_count =
2613                     htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2614                     PDU_TOTAL_TRANSFER_LEN(request));
2615                 //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2616                 //    ntohl(bhssr->bhssr_residual_count));
2617         } else if (PDU_TOTAL_TRANSFER_LEN(request) >
2618             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2619                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2620                 bhssr->bhssr_residual_count =
2621                     htonl(PDU_TOTAL_TRANSFER_LEN(request) -
2622                     ntohl(bhssc->bhssc_expected_data_transfer_length));
2623                 //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2624                 //    ntohl(bhssr->bhssr_residual_count));
2625         }
2626         bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2627         bhssr->bhssr_status = io->scsiio.scsi_status;
2628         bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2629         bhssr->bhssr_expdatasn = htonl(PDU_EXPDATASN(request));
2630
2631         if (io->scsiio.sense_len > 0) {
2632 #if 0
2633                 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2634                     io->scsiio.sense_len);
2635 #endif
2636                 sense_length = htons(io->scsiio.sense_len);
2637                 icl_pdu_append_data(response,
2638                     &sense_length, sizeof(sense_length), M_WAITOK);
2639                 icl_pdu_append_data(response,
2640                     &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2641         }
2642
2643         ctl_free_io(io);
2644         icl_pdu_free(request);
2645         cfiscsi_pdu_queue(response);
2646 }
2647
2648 static void
2649 cfiscsi_task_management_done(union ctl_io *io)
2650 {
2651         struct icl_pdu *request, *response;
2652         struct iscsi_bhs_task_management_request *bhstmr;
2653         struct iscsi_bhs_task_management_response *bhstmr2;
2654         struct cfiscsi_data_wait *cdw, *tmpcdw;
2655         struct cfiscsi_session *cs;
2656
2657         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2658         cs = PDU_SESSION(request);
2659         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2660         KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2661             ISCSI_BHS_OPCODE_TASK_REQUEST,
2662             ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2663
2664 #if 0
2665         CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2666             bhstmr->bhstmr_initiator_task_tag,
2667             bhstmr->bhstmr_referenced_task_tag);
2668 #endif
2669
2670         if ((bhstmr->bhstmr_function & ~0x80) ==
2671             BHSTMR_FUNCTION_ABORT_TASK) {
2672                 /*
2673                  * Make sure we no longer wait for Data-Out for this command.
2674                  */
2675                 CFISCSI_SESSION_LOCK(cs);
2676                 TAILQ_FOREACH_SAFE(cdw,
2677                     &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2678                         if (bhstmr->bhstmr_referenced_task_tag !=
2679                             cdw->cdw_initiator_task_tag)
2680                                 continue;
2681
2682 #if 0
2683                         CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2684                             "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2685 #endif
2686                         TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2687                             cdw, cdw_next);
2688                         cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
2689                         uma_zfree(cfiscsi_data_wait_zone, cdw);
2690                 }
2691                 CFISCSI_SESSION_UNLOCK(cs);
2692         }
2693
2694         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2695         bhstmr2 = (struct iscsi_bhs_task_management_response *)
2696             response->ip_bhs;
2697         bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
2698         bhstmr2->bhstmr_flags = 0x80;
2699         if (io->io_hdr.status == CTL_SUCCESS) {
2700                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
2701         } else {
2702                 /*
2703                  * XXX: How to figure out what exactly went wrong?  iSCSI spec
2704                  *      expects us to provide detailed error, e.g. "Task does
2705                  *      not exist" or "LUN does not exist".
2706                  */
2707                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED");
2708                 bhstmr2->bhstmr_response =
2709                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
2710         }
2711         bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
2712
2713         ctl_free_io(io);
2714         icl_pdu_free(request);
2715         cfiscsi_pdu_queue(response);
2716 }
2717
2718 static void
2719 cfiscsi_done(union ctl_io *io)
2720 {
2721         struct icl_pdu *request;
2722         struct cfiscsi_session *cs;
2723
2724         KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
2725                 ("invalid CTL status %#x", io->io_hdr.status));
2726
2727         request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
2728         if (request == NULL) {
2729                 /*
2730                  * Implicit task termination has just completed; nothing to do.
2731                  */
2732                 return;
2733         }
2734
2735         cs = PDU_SESSION(request);
2736         refcount_release(&cs->cs_outstanding_ctl_pdus);
2737
2738         switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
2739         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
2740                 cfiscsi_scsi_command_done(io);
2741                 break;
2742         case ISCSI_BHS_OPCODE_TASK_REQUEST:
2743                 cfiscsi_task_management_done(io);
2744                 break;
2745         default:
2746                 panic("cfiscsi_done called with wrong opcode 0x%x",
2747                     request->ip_bhs->bhs_opcode);
2748         }
2749 }