]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/cam/ctl/ctl_frontend_iscsi.c
OpenSSL: update to 3.0.12
[FreeBSD/FreeBSD.git] / sys / cam / ctl / ctl_frontend_iscsi.c
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2012 The FreeBSD Foundation
5  *
6  * This software was developed by Edward Tomasz Napierala under sponsorship
7  * from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  */
30
31 /*
32  * CTL frontend for the iSCSI protocol.
33  */
34
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
37 #include <sys/capsicum.h>
38 #include <sys/condvar.h>
39 #include <sys/endian.h>
40 #include <sys/file.h>
41 #include <sys/kernel.h>
42 #include <sys/kthread.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/module.h>
46 #include <sys/mutex.h>
47 #include <sys/queue.h>
48 #include <sys/sbuf.h>
49 #include <sys/socket.h>
50 #include <sys/sysctl.h>
51 #include <sys/systm.h>
52 #include <sys/uio.h>
53 #include <sys/unistd.h>
54 #include <sys/nv.h>
55 #include <sys/dnv.h>
56 #include <vm/uma.h>
57
58 #include <cam/scsi/scsi_all.h>
59 #include <cam/scsi/scsi_da.h>
60 #include <cam/ctl/ctl_io.h>
61 #include <cam/ctl/ctl.h>
62 #include <cam/ctl/ctl_backend.h>
63 #include <cam/ctl/ctl_error.h>
64 #include <cam/ctl/ctl_frontend.h>
65 #include <cam/ctl/ctl_debug.h>
66 #include <cam/ctl/ctl_ha.h>
67 #include <cam/ctl/ctl_ioctl.h>
68 #include <cam/ctl/ctl_private.h>
69
70 #include <dev/iscsi/icl.h>
71 #include <dev/iscsi/icl_wrappers.h>
72 #include <dev/iscsi/iscsi_proto.h>
73 #include <cam/ctl/ctl_frontend_iscsi.h>
74
75 #ifdef ICL_KERNEL_PROXY
76 #include <sys/socketvar.h>
77 #endif
78
79 #ifdef ICL_KERNEL_PROXY
80 FEATURE(cfiscsi_kernel_proxy, "iSCSI target built with ICL_KERNEL_PROXY");
81 #endif
82
83 /* Used for internal nexus reset task. */
84 #define ISCSI_BHS_OPCODE_INTERNAL       0x3e
85
86 static MALLOC_DEFINE(M_CFISCSI, "cfiscsi", "Memory used for CTL iSCSI frontend");
87 static uma_zone_t cfiscsi_data_wait_zone;
88
89 SYSCTL_NODE(_kern_cam_ctl, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
90     "CAM Target Layer iSCSI Frontend");
91 static int debug = 1;
92 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
93     &debug, 1, "Enable debug messages");
94 static int ping_timeout = 5;
95 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN,
96     &ping_timeout, 5, "Interval between ping (NOP-Out) requests, in seconds");
97 static int login_timeout = 60;
98 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN,
99     &login_timeout, 60, "Time to wait for ctld(8) to finish Login Phase, in seconds");
100 static int maxtags = 256;
101 SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN,
102     &maxtags, 0, "Max number of requests queued by initiator");
103
104 #define CFISCSI_DEBUG(X, ...)                                           \
105         do {                                                            \
106                 if (debug > 1) {                                        \
107                         printf("%s: " X "\n",                           \
108                             __func__, ## __VA_ARGS__);                  \
109                 }                                                       \
110         } while (0)
111
112 #define CFISCSI_WARN(X, ...)                                            \
113         do {                                                            \
114                 if (debug > 0) {                                        \
115                         printf("WARNING: %s: " X "\n",                  \
116                             __func__, ## __VA_ARGS__);                  \
117                 }                                                       \
118         } while (0)
119
120 #define CFISCSI_SESSION_DEBUG(S, X, ...)                                \
121         do {                                                            \
122                 if (debug > 1) {                                        \
123                         printf("%s: %s (%s): " X "\n",                  \
124                             __func__, S->cs_initiator_addr,             \
125                             S->cs_initiator_name, ## __VA_ARGS__);      \
126                 }                                                       \
127         } while (0)
128
129 #define CFISCSI_SESSION_WARN(S, X, ...)                                 \
130         do  {                                                           \
131                 if (debug > 0) {                                        \
132                         printf("WARNING: %s (%s): " X "\n",             \
133                             S->cs_initiator_addr,                       \
134                             S->cs_initiator_name, ## __VA_ARGS__);      \
135                 }                                                       \
136         } while (0)
137
138 #define CFISCSI_SESSION_LOCK(X)         mtx_lock(&X->cs_lock)
139 #define CFISCSI_SESSION_UNLOCK(X)       mtx_unlock(&X->cs_lock)
140 #define CFISCSI_SESSION_LOCK_ASSERT(X)  mtx_assert(&X->cs_lock, MA_OWNED)
141
142 #define CONN_SESSION(X)                 ((struct cfiscsi_session *)(X)->ic_prv0)
143 #define PDU_SESSION(X)                  CONN_SESSION((X)->ip_conn)
144
145 struct cfiscsi_priv {
146         void            *request;
147         uint32_t         expdatasn;
148         uint32_t         r2tsn;
149 };
150 #define PRIV(io)        \
151     ((struct cfiscsi_priv *)&(io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND])
152 #define PRIV_REQUEST(io)                PRIV(io)->request
153 #define PRIV_EXPDATASN(io)              PRIV(io)->expdatasn
154 #define PRIV_R2TSN(io)                  PRIV(io)->r2tsn
155
156 static int      cfiscsi_init(void);
157 static int      cfiscsi_shutdown(void);
158 static void     cfiscsi_online(void *arg);
159 static void     cfiscsi_offline(void *arg);
160 static int      cfiscsi_info(void *arg, struct sbuf *sb);
161 static int      cfiscsi_ioctl(struct cdev *dev,
162                     u_long cmd, caddr_t addr, int flag, struct thread *td);
163 static void     cfiscsi_datamove(union ctl_io *io);
164 static void     cfiscsi_datamove_in(union ctl_io *io);
165 static void     cfiscsi_datamove_out(union ctl_io *io);
166 static void     cfiscsi_done(union ctl_io *io);
167 static bool     cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request);
168 static void     cfiscsi_pdu_handle_nop_out(struct icl_pdu *request);
169 static void     cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request);
170 static void     cfiscsi_pdu_handle_task_request(struct icl_pdu *request);
171 static void     cfiscsi_pdu_handle_data_out(struct icl_pdu *request);
172 static void     cfiscsi_pdu_handle_logout_request(struct icl_pdu *request);
173 static void     cfiscsi_session_terminate(struct cfiscsi_session *cs);
174 static struct cfiscsi_data_wait *cfiscsi_data_wait_new(
175                     struct cfiscsi_session *cs, union ctl_io *io,
176                     uint32_t initiator_task_tag,
177                     uint32_t *target_transfer_tagp);
178 static void     cfiscsi_data_wait_free(struct cfiscsi_session *cs,
179                     struct cfiscsi_data_wait *cdw);
180 static struct cfiscsi_target    *cfiscsi_target_find(struct cfiscsi_softc
181                     *softc, const char *name, uint16_t tag);
182 static struct cfiscsi_target    *cfiscsi_target_find_or_create(
183     struct cfiscsi_softc *softc, const char *name, const char *alias,
184     uint16_t tag);
185 static void     cfiscsi_target_release(struct cfiscsi_target *ct);
186 static void     cfiscsi_session_delete(struct cfiscsi_session *cs);
187
188 static struct cfiscsi_softc cfiscsi_softc;
189
190 static struct ctl_frontend cfiscsi_frontend =
191 {
192         .name = "iscsi",
193         .init = cfiscsi_init,
194         .ioctl = cfiscsi_ioctl,
195         .shutdown = cfiscsi_shutdown,
196 };
197 CTL_FRONTEND_DECLARE(cfiscsi, cfiscsi_frontend);
198 MODULE_DEPEND(cfiscsi, icl, 1, 1, 1);
199
200 static struct icl_pdu *
201 cfiscsi_pdu_new_response(struct icl_pdu *request, int flags)
202 {
203
204         return (icl_pdu_new(request->ip_conn, flags));
205 }
206
207 static bool
208 cfiscsi_pdu_update_cmdsn(const struct icl_pdu *request)
209 {
210         const struct iscsi_bhs_scsi_command *bhssc;
211         struct cfiscsi_session *cs;
212         uint32_t cmdsn, curcmdsn;
213
214         cs = PDU_SESSION(request);
215
216         /*
217          * Every incoming PDU - not just NOP-Out - resets the ping timer.
218          * The purpose of the timeout is to reset the connection when it stalls;
219          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
220          * in some queue.
221          */
222         cs->cs_timeout = 0;
223
224         /*
225          * Immediate commands carry cmdsn, but it is neither incremented nor
226          * verified.
227          */
228         if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_IMMEDIATE)
229                 return (false);
230
231         /*
232          * Data-Out PDUs don't contain CmdSN.
233          */
234         if (request->ip_bhs->bhs_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
235                 return (false);
236
237         /*
238          * We're only using fields common for all the request
239          * (initiator -> target) PDUs.
240          */
241         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
242         curcmdsn = cmdsn = ntohl(bhssc->bhssc_cmdsn);
243
244         /*
245          * Increment session cmdsn and exit if we received the expected value.
246          */
247         do {
248                 if (atomic_fcmpset_32(&cs->cs_cmdsn, &curcmdsn, cmdsn + 1))
249                         return (false);
250         } while (curcmdsn == cmdsn);
251
252         /*
253          * The target MUST silently ignore any non-immediate command outside
254          * of this range.
255          */
256         if (ISCSI_SNLT(cmdsn, curcmdsn) ||
257             ISCSI_SNGT(cmdsn, curcmdsn - 1 + maxtags)) {
258                 CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
259                     "while expected %u", cmdsn, curcmdsn);
260                 return (true);
261         }
262
263         /*
264          * We don't support multiple connections now, so any discontinuity in
265          * CmdSN means lost PDUs.  Since we don't support PDU retransmission --
266          * terminate the connection.
267          */
268         CFISCSI_SESSION_WARN(cs, "received PDU with CmdSN %u, "
269             "while expected %u; dropping connection",
270             cmdsn, curcmdsn);
271         cfiscsi_session_terminate(cs);
272         return (true);
273 }
274
275 static void
276 cfiscsi_pdu_handle(struct icl_pdu *request)
277 {
278         struct cfiscsi_session *cs;
279         bool ignore;
280
281         cs = PDU_SESSION(request);
282
283         ignore = cfiscsi_pdu_update_cmdsn(request);
284         if (ignore) {
285                 icl_pdu_free(request);
286                 return;
287         }
288
289         /*
290          * Handle the PDU; this includes e.g. receiving the remaining
291          * part of PDU and submitting the SCSI command to CTL
292          * or queueing a reply.  The handling routine is responsible
293          * for freeing the PDU when it's no longer needed.
294          */
295         switch (request->ip_bhs->bhs_opcode &
296             ~ISCSI_BHS_OPCODE_IMMEDIATE) {
297         case ISCSI_BHS_OPCODE_NOP_OUT:
298                 cfiscsi_pdu_handle_nop_out(request);
299                 break;
300         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
301                 cfiscsi_pdu_handle_scsi_command(request);
302                 break;
303         case ISCSI_BHS_OPCODE_TASK_REQUEST:
304                 cfiscsi_pdu_handle_task_request(request);
305                 break;
306         case ISCSI_BHS_OPCODE_SCSI_DATA_OUT:
307                 cfiscsi_pdu_handle_data_out(request);
308                 break;
309         case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
310                 cfiscsi_pdu_handle_logout_request(request);
311                 break;
312         default:
313                 CFISCSI_SESSION_WARN(cs, "received PDU with unsupported "
314                     "opcode 0x%x; dropping connection",
315                     request->ip_bhs->bhs_opcode);
316                 icl_pdu_free(request);
317                 cfiscsi_session_terminate(cs);
318         }
319
320 }
321
322 static void
323 cfiscsi_receive_callback(struct icl_pdu *request)
324 {
325 #ifdef ICL_KERNEL_PROXY
326         struct cfiscsi_session *cs;
327
328         cs = PDU_SESSION(request);
329         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
330                 if (cs->cs_login_pdu == NULL)
331                         cs->cs_login_pdu = request;
332                 else
333                         icl_pdu_free(request);
334                 cv_signal(&cs->cs_login_cv);
335                 return;
336         }
337 #endif
338
339         cfiscsi_pdu_handle(request);
340 }
341
342 static void
343 cfiscsi_error_callback(struct icl_conn *ic)
344 {
345         struct cfiscsi_session *cs;
346
347         cs = CONN_SESSION(ic);
348
349         CFISCSI_SESSION_WARN(cs, "connection error; dropping connection");
350         cfiscsi_session_terminate(cs);
351 }
352
353 static int
354 cfiscsi_pdu_prepare(struct icl_pdu *response)
355 {
356         struct cfiscsi_session *cs;
357         struct iscsi_bhs_scsi_response *bhssr;
358         bool advance_statsn = true;
359         uint32_t cmdsn;
360
361         cs = PDU_SESSION(response);
362
363         CFISCSI_SESSION_LOCK_ASSERT(cs);
364
365         /*
366          * We're only using fields common for all the response
367          * (target -> initiator) PDUs.
368          */
369         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
370
371         /*
372          * 10.8.3: "The StatSN for this connection is not advanced
373          * after this PDU is sent."
374          */
375         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_R2T)
376                 advance_statsn = false;
377
378         /*
379          * 10.19.2: "However, when the Initiator Task Tag is set to 0xffffffff,
380          * StatSN for the connection is not advanced after this PDU is sent."
381          */
382         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_NOP_IN && 
383             bhssr->bhssr_initiator_task_tag == 0xffffffff)
384                 advance_statsn = false;
385
386         /*
387          * See the comment below - StatSN is not meaningful and must
388          * not be advanced.
389          */
390         if (bhssr->bhssr_opcode == ISCSI_BHS_OPCODE_SCSI_DATA_IN &&
391             (bhssr->bhssr_flags & BHSDI_FLAGS_S) == 0)
392                 advance_statsn = false;
393
394         /*
395          * 10.7.3: "The fields StatSN, Status, and Residual Count
396          * only have meaningful content if the S bit is set to 1."
397          */
398         if (bhssr->bhssr_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
399             (bhssr->bhssr_flags & BHSDI_FLAGS_S))
400                 bhssr->bhssr_statsn = htonl(cs->cs_statsn);
401         cmdsn = cs->cs_cmdsn;
402         bhssr->bhssr_expcmdsn = htonl(cmdsn);
403         bhssr->bhssr_maxcmdsn = htonl(cmdsn - 1 +
404             imax(0, maxtags - cs->cs_outstanding_ctl_pdus));
405
406         if (advance_statsn)
407                 cs->cs_statsn++;
408
409         return (0);
410 }
411
412 static void
413 cfiscsi_pdu_queue(struct icl_pdu *response)
414 {
415         struct cfiscsi_session *cs;
416
417         cs = PDU_SESSION(response);
418
419         CFISCSI_SESSION_LOCK(cs);
420         cfiscsi_pdu_prepare(response);
421         icl_pdu_queue(response);
422         CFISCSI_SESSION_UNLOCK(cs);
423 }
424
425  static void
426 cfiscsi_pdu_queue_cb(struct icl_pdu *response, icl_pdu_cb cb)
427 {
428         struct cfiscsi_session *cs = PDU_SESSION(response);
429
430         CFISCSI_SESSION_LOCK(cs);
431         cfiscsi_pdu_prepare(response);
432         icl_pdu_queue_cb(response, cb);
433         CFISCSI_SESSION_UNLOCK(cs);
434 }
435
436 static void
437 cfiscsi_pdu_handle_nop_out(struct icl_pdu *request)
438 {
439         struct cfiscsi_session *cs;
440         struct iscsi_bhs_nop_out *bhsno;
441         struct iscsi_bhs_nop_in *bhsni;
442         struct icl_pdu *response;
443         void *data = NULL;
444         size_t datasize;
445         int error;
446
447         cs = PDU_SESSION(request);
448         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
449
450         if (bhsno->bhsno_initiator_task_tag == 0xffffffff) {
451                 /*
452                  * Nothing to do, iscsi_pdu_update_statsn() already
453                  * zeroed the timeout.
454                  */
455                 icl_pdu_free(request);
456                 return;
457         }
458
459         datasize = icl_pdu_data_segment_length(request);
460         if (datasize > 0) {
461                 data = malloc(datasize, M_CFISCSI, M_NOWAIT | M_ZERO);
462                 if (data == NULL) {
463                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
464                             "dropping connection");
465                         icl_pdu_free(request);
466                         cfiscsi_session_terminate(cs);
467                         return;
468                 }
469                 icl_pdu_get_data(request, 0, data, datasize);
470         }
471
472         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
473         if (response == NULL) {
474                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
475                     "droppping connection");
476                 free(data, M_CFISCSI);
477                 icl_pdu_free(request);
478                 cfiscsi_session_terminate(cs);
479                 return;
480         }
481         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
482         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
483         bhsni->bhsni_flags = 0x80;
484         bhsni->bhsni_initiator_task_tag = bhsno->bhsno_initiator_task_tag;
485         bhsni->bhsni_target_transfer_tag = 0xffffffff;
486         if (datasize > 0) {
487                 error = icl_pdu_append_data(response, data, datasize, M_NOWAIT);
488                 if (error != 0) {
489                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
490                             "dropping connection");
491                         free(data, M_CFISCSI);
492                         icl_pdu_free(request);
493                         icl_pdu_free(response);
494                         cfiscsi_session_terminate(cs);
495                         return;
496                 }
497                 free(data, M_CFISCSI);
498         }
499
500         icl_pdu_free(request);
501         cfiscsi_pdu_queue(response);
502 }
503
504 static void
505 cfiscsi_pdu_handle_scsi_command(struct icl_pdu *request)
506 {
507         struct iscsi_bhs_scsi_command *bhssc;
508         struct cfiscsi_session *cs;
509         union ctl_io *io;
510         int error;
511
512         cs = PDU_SESSION(request);
513         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
514         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
515         //    bhssc->bhssc_initiator_task_tag);
516
517         if (request->ip_data_len > 0 && cs->cs_immediate_data == false) {
518                 CFISCSI_SESSION_WARN(cs, "unsolicited data with "
519                     "ImmediateData=No; dropping connection");
520                 icl_pdu_free(request);
521                 cfiscsi_session_terminate(cs);
522                 return;
523         }
524         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
525         ctl_zero_io(io);
526         PRIV_REQUEST(io) = request;
527         io->io_hdr.io_type = CTL_IO_SCSI;
528         io->io_hdr.nexus.initid = cs->cs_ctl_initid;
529         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
530         io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhssc->bhssc_lun));
531         io->scsiio.priority = (bhssc->bhssc_pri & BHSSC_PRI_MASK) >>
532             BHSSC_PRI_SHIFT;
533         io->scsiio.tag_num = bhssc->bhssc_initiator_task_tag;
534         switch ((bhssc->bhssc_flags & BHSSC_FLAGS_ATTR)) {
535         case BHSSC_FLAGS_ATTR_UNTAGGED:
536                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
537                 break;
538         case BHSSC_FLAGS_ATTR_SIMPLE:
539                 io->scsiio.tag_type = CTL_TAG_SIMPLE;
540                 break;
541         case BHSSC_FLAGS_ATTR_ORDERED:
542                 io->scsiio.tag_type = CTL_TAG_ORDERED;
543                 break;
544         case BHSSC_FLAGS_ATTR_HOQ:
545                 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
546                 break;
547         case BHSSC_FLAGS_ATTR_ACA:
548                 io->scsiio.tag_type = CTL_TAG_ACA;
549                 break;
550         default:
551                 io->scsiio.tag_type = CTL_TAG_UNTAGGED;
552                 CFISCSI_SESSION_WARN(cs, "unhandled tag type %d",
553                     bhssc->bhssc_flags & BHSSC_FLAGS_ATTR);
554                 break;
555         }
556         io->scsiio.cdb_len = sizeof(bhssc->bhssc_cdb); /* Which is 16. */
557         memcpy(io->scsiio.cdb, bhssc->bhssc_cdb, sizeof(bhssc->bhssc_cdb));
558         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
559         error = ctl_run(io);
560         if (error != CTL_RETVAL_COMPLETE) {
561                 CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "
562                     "dropping connection", error);
563                 ctl_free_io(io);
564                 refcount_release(&cs->cs_outstanding_ctl_pdus);
565                 icl_pdu_free(request);
566                 cfiscsi_session_terminate(cs);
567         }
568 }
569
570 static void
571 cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
572 {
573         struct iscsi_bhs_task_management_request *bhstmr;
574         struct iscsi_bhs_task_management_response *bhstmr2;
575         struct icl_pdu *response;
576         struct cfiscsi_session *cs;
577         union ctl_io *io;
578         int error;
579
580         cs = PDU_SESSION(request);
581         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
582         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
583         ctl_zero_io(io);
584         PRIV_REQUEST(io) = request;
585         io->io_hdr.io_type = CTL_IO_TASK;
586         io->io_hdr.nexus.initid = cs->cs_ctl_initid;
587         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
588         io->io_hdr.nexus.targ_lun = ctl_decode_lun(be64toh(bhstmr->bhstmr_lun));
589         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
590
591         switch (bhstmr->bhstmr_function & ~0x80) {
592         case BHSTMR_FUNCTION_ABORT_TASK:
593 #if 0
594                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK");
595 #endif
596                 io->taskio.task_action = CTL_TASK_ABORT_TASK;
597                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
598                 break;
599         case BHSTMR_FUNCTION_ABORT_TASK_SET:
600 #if 0
601                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_ABORT_TASK_SET");
602 #endif
603                 io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
604                 break;
605         case BHSTMR_FUNCTION_CLEAR_TASK_SET:
606 #if 0
607                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
608 #endif
609                 io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
610                 break;
611         case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
612 #if 0
613                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
614 #endif
615                 io->taskio.task_action = CTL_TASK_LUN_RESET;
616                 break;
617         case BHSTMR_FUNCTION_TARGET_WARM_RESET:
618 #if 0
619                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_WARM_RESET");
620 #endif
621                 io->taskio.task_action = CTL_TASK_TARGET_RESET;
622                 break;
623         case BHSTMR_FUNCTION_TARGET_COLD_RESET:
624 #if 0
625                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
626 #endif
627                 io->taskio.task_action = CTL_TASK_TARGET_RESET;
628                 break;
629         case BHSTMR_FUNCTION_QUERY_TASK:
630 #if 0
631                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
632 #endif
633                 io->taskio.task_action = CTL_TASK_QUERY_TASK;
634                 io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
635                 break;
636         case BHSTMR_FUNCTION_QUERY_TASK_SET:
637 #if 0
638                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
639 #endif
640                 io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
641                 break;
642         case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
643 #if 0
644                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
645 #endif
646                 io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
647                 break;
648         case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
649 #if 0
650                 CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
651 #endif
652                 io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
653                 break;
654         default:
655                 CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
656                     bhstmr->bhstmr_function & ~0x80);
657                 ctl_free_io(io);
658
659                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
660                 if (response == NULL) {
661                         CFISCSI_SESSION_WARN(cs, "failed to allocate memory; "
662                             "dropping connection");
663                         icl_pdu_free(request);
664                         cfiscsi_session_terminate(cs);
665                         return;
666                 }
667                 bhstmr2 = (struct iscsi_bhs_task_management_response *)
668                     response->ip_bhs;
669                 bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
670                 bhstmr2->bhstmr_flags = 0x80;
671                 bhstmr2->bhstmr_response =
672                     BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
673                 bhstmr2->bhstmr_initiator_task_tag =
674                     bhstmr->bhstmr_initiator_task_tag;
675                 icl_pdu_free(request);
676                 cfiscsi_pdu_queue(response);
677                 return;
678         }
679
680         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
681         error = ctl_run(io);
682         if (error != CTL_RETVAL_COMPLETE) {
683                 CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d; "
684                     "dropping connection", error);
685                 ctl_free_io(io);
686                 refcount_release(&cs->cs_outstanding_ctl_pdus);
687                 icl_pdu_free(request);
688                 cfiscsi_session_terminate(cs);
689         }
690 }
691
692 static bool
693 cfiscsi_handle_data_segment(struct icl_pdu *request, struct cfiscsi_data_wait *cdw)
694 {
695         struct iscsi_bhs_data_out *bhsdo;
696         struct cfiscsi_session *cs;
697         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
698         size_t copy_len, len, off, buffer_offset;
699         int ctl_sg_count;
700         union ctl_io *io;
701
702         cs = PDU_SESSION(request);
703
704         KASSERT((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
705             ISCSI_BHS_OPCODE_SCSI_DATA_OUT ||
706             (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
707             ISCSI_BHS_OPCODE_SCSI_COMMAND,
708             ("bad opcode 0x%x", request->ip_bhs->bhs_opcode));
709
710         /*
711          * We're only using fields common for Data-Out and SCSI Command PDUs.
712          */
713         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
714
715         io = cdw->cdw_ctl_io;
716         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
717             ("CTL_FLAG_DATA_IN"));
718
719 #if 0
720         CFISCSI_SESSION_DEBUG(cs, "received %zd bytes out of %d",
721             request->ip_data_len, io->scsiio.kern_total_len);
722 #endif
723
724         if (io->scsiio.kern_sg_entries > 0) {
725                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
726                 ctl_sg_count = io->scsiio.kern_sg_entries;
727         } else {
728                 ctl_sglist = &ctl_sg_entry;
729                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
730                 ctl_sglist->len = io->scsiio.kern_data_len;
731                 ctl_sg_count = 1;
732         }
733
734         if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
735             ISCSI_BHS_OPCODE_SCSI_DATA_OUT)
736                 buffer_offset = ntohl(bhsdo->bhsdo_buffer_offset);
737         else
738                 buffer_offset = 0;
739         len = icl_pdu_data_segment_length(request);
740
741         /*
742          * Make sure the offset, as sent by the initiator, matches the offset
743          * we're supposed to be at in the scatter-gather list.
744          */
745         if (buffer_offset >
746             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled ||
747             buffer_offset + len <=
748             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled) {
749                 CFISCSI_SESSION_WARN(cs, "received bad buffer offset %zd, "
750                     "expected %zd; dropping connection", buffer_offset,
751                     (size_t)io->scsiio.kern_rel_offset +
752                     (size_t)io->scsiio.ext_data_filled);
753                 ctl_set_data_phase_error(&io->scsiio);
754                 cfiscsi_session_terminate(cs);
755                 return (true);
756         }
757
758         /*
759          * This is the offset within the PDU data segment, as opposed
760          * to buffer_offset, which is the offset within the task (SCSI
761          * command).
762          */
763         off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled -
764             buffer_offset;
765
766         /*
767          * Iterate over the scatter/gather segments, filling them with data
768          * from the PDU data segment.  Note that this can get called multiple
769          * times for one SCSI command; the cdw structure holds state for the
770          * scatter/gather list.
771          */
772         for (;;) {
773                 KASSERT(cdw->cdw_sg_index < ctl_sg_count,
774                     ("cdw->cdw_sg_index >= ctl_sg_count"));
775                 if (cdw->cdw_sg_len == 0) {
776                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
777                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
778                 }
779                 KASSERT(off <= len, ("len > off"));
780                 copy_len = len - off;
781                 if (copy_len > cdw->cdw_sg_len)
782                         copy_len = cdw->cdw_sg_len;
783
784                 icl_pdu_get_data(request, off, cdw->cdw_sg_addr, copy_len);
785                 cdw->cdw_sg_addr += copy_len;
786                 cdw->cdw_sg_len -= copy_len;
787                 off += copy_len;
788                 io->scsiio.ext_data_filled += copy_len;
789                 io->scsiio.kern_data_resid -= copy_len;
790
791                 if (cdw->cdw_sg_len == 0) {
792                         /*
793                          * End of current segment.
794                          */
795                         if (cdw->cdw_sg_index == ctl_sg_count - 1) {
796                                 /*
797                                  * Last segment in scatter/gather list.
798                                  */
799                                 break;
800                         }
801                         cdw->cdw_sg_index++;
802                 }
803
804                 if (off == len) {
805                         /*
806                          * End of PDU payload.
807                          */
808                         break;
809                 }
810         }
811
812         if (len > off) {
813                 /*
814                  * In case of unsolicited data, it's possible that the buffer
815                  * provided by CTL is smaller than negotiated FirstBurstLength.
816                  * Just ignore the superfluous data; will ask for them with R2T
817                  * on next call to cfiscsi_datamove().
818                  *
819                  * This obviously can only happen with SCSI Command PDU. 
820                  */
821                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
822                     ISCSI_BHS_OPCODE_SCSI_COMMAND)
823                         return (true);
824
825                 CFISCSI_SESSION_WARN(cs, "received too much data: got %zd bytes, "
826                     "expected %zd; dropping connection",
827                     icl_pdu_data_segment_length(request), off);
828                 ctl_set_data_phase_error(&io->scsiio);
829                 cfiscsi_session_terminate(cs);
830                 return (true);
831         }
832
833         if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end &&
834             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) == 0) {
835                 CFISCSI_SESSION_WARN(cs, "got the final packet without "
836                     "the F flag; flags = 0x%x; dropping connection",
837                     bhsdo->bhsdo_flags);
838                 ctl_set_data_phase_error(&io->scsiio);
839                 cfiscsi_session_terminate(cs);
840                 return (true);
841         }
842
843         if (io->scsiio.ext_data_filled != cdw->cdw_r2t_end &&
844             (bhsdo->bhsdo_flags & BHSDO_FLAGS_F) != 0) {
845                 if ((request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
846                     ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
847                         CFISCSI_SESSION_WARN(cs, "got the final packet, but the "
848                             "transmitted size was %zd bytes instead of %d; "
849                             "dropping connection",
850                             (size_t)io->scsiio.ext_data_filled,
851                             cdw->cdw_r2t_end);
852                         ctl_set_data_phase_error(&io->scsiio);
853                         cfiscsi_session_terminate(cs);
854                         return (true);
855                 } else {
856                         /*
857                          * For SCSI Command PDU, this just means we need to
858                          * solicit more data by sending R2T.
859                          */
860                         return (false);
861                 }
862         }
863
864         if (io->scsiio.ext_data_filled == cdw->cdw_r2t_end) {
865 #if 0
866                 CFISCSI_SESSION_DEBUG(cs, "no longer expecting Data-Out with target "
867                     "transfer tag 0x%x", cdw->cdw_target_transfer_tag);
868 #endif
869
870                 return (true);
871         }
872
873         return (false);
874 }
875
876 static void
877 cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
878 {
879         struct iscsi_bhs_data_out *bhsdo;
880         struct cfiscsi_session *cs;
881         struct cfiscsi_data_wait *cdw = NULL;
882         union ctl_io *io;
883         bool done;
884
885         cs = PDU_SESSION(request);
886         bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
887
888         CFISCSI_SESSION_LOCK(cs);
889         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next) {
890 #if 0
891                 CFISCSI_SESSION_DEBUG(cs, "have ttt 0x%x, itt 0x%x; looking for "
892                     "ttt 0x%x, itt 0x%x",
893                     bhsdo->bhsdo_target_transfer_tag,
894                     bhsdo->bhsdo_initiator_task_tag,
895                     cdw->cdw_target_transfer_tag, cdw->cdw_initiator_task_tag));
896 #endif
897                 if (bhsdo->bhsdo_target_transfer_tag ==
898                     cdw->cdw_target_transfer_tag)
899                         break;
900         }
901         CFISCSI_SESSION_UNLOCK(cs);
902         if (cdw == NULL) {
903                 CFISCSI_SESSION_WARN(cs, "data transfer tag 0x%x, initiator task tag "
904                     "0x%x, not found; dropping connection",
905                     bhsdo->bhsdo_target_transfer_tag, bhsdo->bhsdo_initiator_task_tag);
906                 icl_pdu_free(request);
907                 cfiscsi_session_terminate(cs);
908                 return;
909         }
910
911         if (cdw->cdw_datasn != ntohl(bhsdo->bhsdo_datasn)) {
912                 CFISCSI_SESSION_WARN(cs, "received Data-Out PDU with "
913                     "DataSN %u, while expected %u; dropping connection",
914                     ntohl(bhsdo->bhsdo_datasn), cdw->cdw_datasn);
915                 icl_pdu_free(request);
916                 cfiscsi_session_terminate(cs);
917                 return;
918         }
919         cdw->cdw_datasn += request->ip_additional_pdus + 1;
920
921         io = cdw->cdw_ctl_io;
922         KASSERT((io->io_hdr.flags & CTL_FLAG_DATA_MASK) != CTL_FLAG_DATA_IN,
923             ("CTL_FLAG_DATA_IN"));
924
925         done = cfiscsi_handle_data_segment(request, cdw);
926         if (done) {
927                 CFISCSI_SESSION_LOCK(cs);
928                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
929                 CFISCSI_SESSION_UNLOCK(cs);
930                 done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
931                     io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
932                 cfiscsi_data_wait_free(cs, cdw);
933                 io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
934                 if (done)
935                         ctl_datamove_done(io, false);
936                 else
937                         cfiscsi_datamove_out(io);
938         }
939
940         icl_pdu_free(request);
941 }
942
943 static void
944 cfiscsi_pdu_handle_logout_request(struct icl_pdu *request)
945 {
946         struct iscsi_bhs_logout_request *bhslr;
947         struct iscsi_bhs_logout_response *bhslr2;
948         struct icl_pdu *response;
949         struct cfiscsi_session *cs;
950
951         cs = PDU_SESSION(request);
952         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
953         switch (bhslr->bhslr_reason & 0x7f) {
954         case BHSLR_REASON_CLOSE_SESSION:
955         case BHSLR_REASON_CLOSE_CONNECTION:
956                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
957                 if (response == NULL) {
958                         CFISCSI_SESSION_DEBUG(cs, "failed to allocate memory");
959                         icl_pdu_free(request);
960                         cfiscsi_session_terminate(cs);
961                         return;
962                 }
963                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
964                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
965                 bhslr2->bhslr_flags = 0x80;
966                 bhslr2->bhslr_response = BHSLR_RESPONSE_CLOSED_SUCCESSFULLY;
967                 bhslr2->bhslr_initiator_task_tag =
968                     bhslr->bhslr_initiator_task_tag;
969                 icl_pdu_free(request);
970                 cfiscsi_pdu_queue(response);
971                 cfiscsi_session_terminate(cs);
972                 break;
973         case BHSLR_REASON_REMOVE_FOR_RECOVERY:
974                 response = cfiscsi_pdu_new_response(request, M_NOWAIT);
975                 if (response == NULL) {
976                         CFISCSI_SESSION_WARN(cs,
977                             "failed to allocate memory; dropping connection");
978                         icl_pdu_free(request);
979                         cfiscsi_session_terminate(cs);
980                         return;
981                 }
982                 bhslr2 = (struct iscsi_bhs_logout_response *)response->ip_bhs;
983                 bhslr2->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_RESPONSE;
984                 bhslr2->bhslr_flags = 0x80;
985                 bhslr2->bhslr_response = BHSLR_RESPONSE_RECOVERY_NOT_SUPPORTED;
986                 bhslr2->bhslr_initiator_task_tag =
987                     bhslr->bhslr_initiator_task_tag;
988                 icl_pdu_free(request);
989                 cfiscsi_pdu_queue(response);
990                 break;
991         default:
992                 CFISCSI_SESSION_WARN(cs, "invalid reason 0%x; dropping connection",
993                     bhslr->bhslr_reason);
994                 icl_pdu_free(request);
995                 cfiscsi_session_terminate(cs);
996                 break;
997         }
998 }
999
1000 static void
1001 cfiscsi_callout(void *context)
1002 {
1003         struct icl_pdu *cp;
1004         struct iscsi_bhs_nop_in *bhsni;
1005         struct cfiscsi_session *cs;
1006
1007         cs = context;
1008
1009         if (cs->cs_terminating) 
1010                 return;
1011
1012         callout_schedule(&cs->cs_callout, 1 * hz);
1013
1014         atomic_add_int(&cs->cs_timeout, 1);
1015
1016 #ifdef ICL_KERNEL_PROXY
1017         if (cs->cs_waiting_for_ctld || cs->cs_login_phase) {
1018                 if (login_timeout > 0 && cs->cs_timeout > login_timeout) {
1019                         CFISCSI_SESSION_WARN(cs, "login timed out after "
1020                             "%d seconds; dropping connection", cs->cs_timeout);
1021                         cfiscsi_session_terminate(cs);
1022                 }
1023                 return;
1024         }
1025 #endif
1026
1027         if (ping_timeout <= 0) {
1028                 /*
1029                  * Pings are disabled.  Don't send NOP-In in this case;
1030                  * user might have disabled pings to work around problems
1031                  * with certain initiators that can't properly handle
1032                  * NOP-In, such as iPXE.  Reset the timeout, to avoid
1033                  * triggering reconnection, should the user decide to
1034                  * reenable them.
1035                  */
1036                 cs->cs_timeout = 0;
1037                 return;
1038         }
1039
1040         if (cs->cs_timeout >= ping_timeout) {
1041                 CFISCSI_SESSION_WARN(cs, "no ping reply (NOP-Out) after %d seconds; "
1042                     "dropping connection",  ping_timeout);
1043                 cfiscsi_session_terminate(cs);
1044                 return;
1045         }
1046
1047         /*
1048          * If the ping was reset less than one second ago - which means
1049          * that we've received some PDU during the last second - assume
1050          * the traffic flows correctly and don't bother sending a NOP-Out.
1051          *
1052          * (It's 2 - one for one second, and one for incrementing is_timeout
1053          * earlier in this routine.)
1054          */
1055         if (cs->cs_timeout < 2)
1056                 return;
1057
1058         cp = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1059         if (cp == NULL) {
1060                 CFISCSI_SESSION_WARN(cs, "failed to allocate memory");
1061                 return;
1062         }
1063         bhsni = (struct iscsi_bhs_nop_in *)cp->ip_bhs;
1064         bhsni->bhsni_opcode = ISCSI_BHS_OPCODE_NOP_IN;
1065         bhsni->bhsni_flags = 0x80;
1066         bhsni->bhsni_initiator_task_tag = 0xffffffff;
1067
1068         cfiscsi_pdu_queue(cp);
1069 }
1070
1071 static struct cfiscsi_data_wait *
1072 cfiscsi_data_wait_new(struct cfiscsi_session *cs, union ctl_io *io,
1073     uint32_t initiator_task_tag, uint32_t *target_transfer_tagp)
1074 {
1075         struct cfiscsi_data_wait *cdw;
1076         int error;
1077
1078         cdw = uma_zalloc(cfiscsi_data_wait_zone, M_NOWAIT | M_ZERO);
1079         if (cdw == NULL) {
1080                 CFISCSI_SESSION_WARN(cs,
1081                     "failed to allocate %zd bytes", sizeof(*cdw));
1082                 return (NULL);
1083         }
1084
1085         error = icl_conn_transfer_setup(cs->cs_conn, PRIV_REQUEST(io), io,
1086             target_transfer_tagp, &cdw->cdw_icl_prv);
1087         if (error != 0) {
1088                 CFISCSI_SESSION_WARN(cs,
1089                     "icl_conn_transfer_setup() failed with error %d", error);
1090                 uma_zfree(cfiscsi_data_wait_zone, cdw);
1091                 return (NULL);
1092         }
1093
1094         cdw->cdw_ctl_io = io;
1095         cdw->cdw_target_transfer_tag = *target_transfer_tagp;
1096         cdw->cdw_initiator_task_tag = initiator_task_tag;
1097
1098         return (cdw);
1099 }
1100
1101 static void
1102 cfiscsi_data_wait_free(struct cfiscsi_session *cs,
1103     struct cfiscsi_data_wait *cdw)
1104 {
1105
1106         icl_conn_transfer_done(cs->cs_conn, cdw->cdw_icl_prv);
1107         uma_zfree(cfiscsi_data_wait_zone, cdw);
1108 }
1109
1110 static void
1111 cfiscsi_data_wait_abort(struct cfiscsi_session *cs,
1112     struct cfiscsi_data_wait *cdw, int status)
1113 {
1114         union ctl_io *cdw_io;
1115
1116         /*
1117          * Set nonzero port status; this prevents backends from
1118          * assuming that the data transfer actually succeeded
1119          * and writing uninitialized data to disk.
1120          */
1121         MPASS(status != 0);
1122         cdw_io = cdw->cdw_ctl_io;
1123         cdw_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
1124         cdw_io->scsiio.io_hdr.port_status = status;
1125         cfiscsi_data_wait_free(cs, cdw);
1126         ctl_datamove_done(cdw_io, false);
1127 }
1128
1129 static void
1130 cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
1131 {
1132         struct cfiscsi_data_wait *cdw;
1133         struct icl_pdu *ip;
1134         union ctl_io *io;
1135         int error, last, wait;
1136
1137         if (cs->cs_target == NULL)
1138                 return;         /* No target yet, so nothing to do. */
1139         ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
1140         ip->ip_bhs->bhs_opcode = ISCSI_BHS_OPCODE_INTERNAL;
1141         io = ctl_alloc_io(cs->cs_target->ct_port.ctl_pool_ref);
1142         ctl_zero_io(io);
1143         PRIV_REQUEST(io) = ip;
1144         io->io_hdr.io_type = CTL_IO_TASK;
1145         io->io_hdr.nexus.initid = cs->cs_ctl_initid;
1146         io->io_hdr.nexus.targ_port = cs->cs_target->ct_port.targ_port;
1147         io->io_hdr.nexus.targ_lun = 0;
1148         io->taskio.tag_type = CTL_TAG_SIMPLE; /* XXX */
1149         io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
1150         wait = cs->cs_outstanding_ctl_pdus;
1151         refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1152         error = ctl_run(io);
1153         if (error != CTL_RETVAL_COMPLETE) {
1154                 CFISCSI_SESSION_WARN(cs, "ctl_run() failed; error %d", error);
1155                 refcount_release(&cs->cs_outstanding_ctl_pdus);
1156                 ctl_free_io(io);
1157                 icl_pdu_free(ip);
1158         }
1159
1160         CFISCSI_SESSION_LOCK(cs);
1161         cs->cs_terminating_tasks = true;
1162         while ((cdw = TAILQ_FIRST(&cs->cs_waiting_for_data_out)) != NULL) {
1163                 TAILQ_REMOVE(&cs->cs_waiting_for_data_out, cdw, cdw_next);
1164                 CFISCSI_SESSION_UNLOCK(cs);
1165                 cfiscsi_data_wait_abort(cs, cdw, 42);
1166                 CFISCSI_SESSION_LOCK(cs);
1167         }
1168         CFISCSI_SESSION_UNLOCK(cs);
1169
1170         /*
1171          * Wait for CTL to terminate all the tasks.
1172          */
1173         if (wait > 0)
1174                 CFISCSI_SESSION_WARN(cs,
1175                     "waiting for CTL to terminate %d tasks", wait);
1176         for (;;) {
1177                 refcount_acquire(&cs->cs_outstanding_ctl_pdus);
1178                 last = refcount_release(&cs->cs_outstanding_ctl_pdus);
1179                 if (last != 0)
1180                         break;
1181                 tsleep(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus),
1182                     0, "cfiscsi_terminate", hz / 100);
1183         }
1184         if (wait > 0)
1185                 CFISCSI_SESSION_WARN(cs, "tasks terminated");
1186 }
1187
1188 static void
1189 cfiscsi_maintenance_thread(void *arg)
1190 {
1191         struct cfiscsi_session *cs;
1192
1193         cs = arg;
1194
1195         for (;;) {
1196                 CFISCSI_SESSION_LOCK(cs);
1197                 if (cs->cs_terminating == false || cs->cs_handoff_in_progress)
1198                         cv_wait(&cs->cs_maintenance_cv, &cs->cs_lock);
1199                 CFISCSI_SESSION_UNLOCK(cs);
1200
1201                 if (cs->cs_terminating && cs->cs_handoff_in_progress == false) {
1202                         /*
1203                          * We used to wait up to 30 seconds to deliver queued
1204                          * PDUs to the initiator.  We also tried hard to deliver
1205                          * SCSI Responses for the aborted PDUs.  We don't do
1206                          * that anymore.  We might need to revisit that.
1207                          */
1208                         callout_drain(&cs->cs_callout);
1209                         icl_conn_close(cs->cs_conn);
1210
1211                         /*
1212                          * At this point ICL receive thread is no longer
1213                          * running; no new tasks can be queued.
1214                          */
1215                         cfiscsi_session_terminate_tasks(cs);
1216                         cfiscsi_session_delete(cs);
1217                         kthread_exit();
1218                         return;
1219                 }
1220                 CFISCSI_SESSION_DEBUG(cs, "nothing to do");
1221         }
1222 }
1223
1224 static void
1225 cfiscsi_session_terminate(struct cfiscsi_session *cs)
1226 {
1227
1228         cs->cs_terminating = true;
1229         cv_signal(&cs->cs_maintenance_cv);
1230 #ifdef ICL_KERNEL_PROXY
1231         cv_signal(&cs->cs_login_cv);
1232 #endif
1233 }
1234
1235 static int
1236 cfiscsi_session_register_initiator(struct cfiscsi_session *cs)
1237 {
1238         struct cfiscsi_target *ct;
1239         char *name;
1240         int i;
1241
1242         KASSERT(cs->cs_ctl_initid == -1, ("already registered"));
1243
1244         ct = cs->cs_target;
1245         name = strdup(cs->cs_initiator_id, M_CTL);
1246         i = ctl_add_initiator(&ct->ct_port, -1, 0, name);
1247         if (i < 0) {
1248                 CFISCSI_SESSION_WARN(cs, "ctl_add_initiator failed with error %d",
1249                     i);
1250                 cs->cs_ctl_initid = -1;
1251                 return (1);
1252         }
1253         cs->cs_ctl_initid = i;
1254 #if 0
1255         CFISCSI_SESSION_DEBUG(cs, "added initiator id %d", i);
1256 #endif
1257
1258         return (0);
1259 }
1260
1261 static void
1262 cfiscsi_session_unregister_initiator(struct cfiscsi_session *cs)
1263 {
1264         int error;
1265
1266         if (cs->cs_ctl_initid == -1)
1267                 return;
1268
1269         error = ctl_remove_initiator(&cs->cs_target->ct_port, cs->cs_ctl_initid);
1270         if (error != 0) {
1271                 CFISCSI_SESSION_WARN(cs, "ctl_remove_initiator failed with error %d",
1272                     error);
1273         }
1274         cs->cs_ctl_initid = -1;
1275 }
1276
1277 static struct cfiscsi_session *
1278 cfiscsi_session_new(struct cfiscsi_softc *softc, const char *offload)
1279 {
1280         struct cfiscsi_session *cs;
1281         int error;
1282
1283         cs = malloc(sizeof(*cs), M_CFISCSI, M_NOWAIT | M_ZERO);
1284         if (cs == NULL) {
1285                 CFISCSI_WARN("malloc failed");
1286                 return (NULL);
1287         }
1288         cs->cs_ctl_initid = -1;
1289
1290         refcount_init(&cs->cs_outstanding_ctl_pdus, 0);
1291         TAILQ_INIT(&cs->cs_waiting_for_data_out);
1292         mtx_init(&cs->cs_lock, "cfiscsi_lock", NULL, MTX_DEF);
1293         cv_init(&cs->cs_maintenance_cv, "cfiscsi_mt");
1294 #ifdef ICL_KERNEL_PROXY
1295         cv_init(&cs->cs_login_cv, "cfiscsi_login");
1296 #endif
1297
1298         /*
1299          * The purpose of this is to avoid racing with session shutdown.
1300          * Otherwise we could have the maintenance thread call icl_conn_close()
1301          * before we call icl_conn_handoff().
1302          */
1303         cs->cs_handoff_in_progress = true;
1304
1305         cs->cs_conn = icl_new_conn(offload, false, "cfiscsi", &cs->cs_lock);
1306         if (cs->cs_conn == NULL) {
1307                 free(cs, M_CFISCSI);
1308                 return (NULL);
1309         }
1310         cs->cs_conn->ic_receive = cfiscsi_receive_callback;
1311         cs->cs_conn->ic_error = cfiscsi_error_callback;
1312         cs->cs_conn->ic_prv0 = cs;
1313
1314         error = kthread_add(cfiscsi_maintenance_thread, cs, NULL, NULL, 0, 0, "cfiscsimt");
1315         if (error != 0) {
1316                 CFISCSI_SESSION_WARN(cs, "kthread_add(9) failed with error %d", error);
1317                 free(cs, M_CFISCSI);
1318                 return (NULL);
1319         }
1320
1321         mtx_lock(&softc->lock);
1322         cs->cs_id = ++softc->last_session_id;
1323         TAILQ_INSERT_TAIL(&softc->sessions, cs, cs_next);
1324         mtx_unlock(&softc->lock);
1325
1326         /*
1327          * Start pinging the initiator.
1328          */
1329         callout_init(&cs->cs_callout, 1);
1330         callout_reset(&cs->cs_callout, 1 * hz, cfiscsi_callout, cs);
1331
1332         return (cs);
1333 }
1334
1335 static void
1336 cfiscsi_session_delete(struct cfiscsi_session *cs)
1337 {
1338         struct cfiscsi_softc *softc;
1339
1340         softc = &cfiscsi_softc;
1341
1342         KASSERT(cs->cs_outstanding_ctl_pdus == 0,
1343             ("destroying session with outstanding CTL pdus"));
1344         KASSERT(TAILQ_EMPTY(&cs->cs_waiting_for_data_out),
1345             ("destroying session with non-empty queue"));
1346
1347         mtx_lock(&softc->lock);
1348         TAILQ_REMOVE(&softc->sessions, cs, cs_next);
1349         mtx_unlock(&softc->lock);
1350
1351         cfiscsi_session_unregister_initiator(cs);
1352         if (cs->cs_target != NULL)
1353                 cfiscsi_target_release(cs->cs_target);
1354         icl_conn_close(cs->cs_conn);
1355         icl_conn_free(cs->cs_conn);
1356         free(cs, M_CFISCSI);
1357         cv_signal(&softc->sessions_cv);
1358 }
1359
1360 static int
1361 cfiscsi_init(void)
1362 {
1363         struct cfiscsi_softc *softc;
1364
1365         softc = &cfiscsi_softc;
1366         bzero(softc, sizeof(*softc));
1367         mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
1368
1369         cv_init(&softc->sessions_cv, "cfiscsi_sessions");
1370 #ifdef ICL_KERNEL_PROXY
1371         cv_init(&softc->accept_cv, "cfiscsi_accept");
1372 #endif
1373         TAILQ_INIT(&softc->sessions);
1374         TAILQ_INIT(&softc->targets);
1375
1376         cfiscsi_data_wait_zone = uma_zcreate("cfiscsi_data_wait",
1377             sizeof(struct cfiscsi_data_wait), NULL, NULL, NULL, NULL,
1378             UMA_ALIGN_PTR, 0);
1379
1380         return (0);
1381 }
1382
1383 static int
1384 cfiscsi_shutdown(void)
1385 {
1386         struct cfiscsi_softc *softc = &cfiscsi_softc;
1387
1388         if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
1389                 return (EBUSY);
1390
1391         uma_zdestroy(cfiscsi_data_wait_zone);
1392 #ifdef ICL_KERNEL_PROXY
1393         cv_destroy(&softc->accept_cv);
1394 #endif
1395         cv_destroy(&softc->sessions_cv);
1396         mtx_destroy(&softc->lock);
1397         return (0);
1398 }
1399
1400 #ifdef ICL_KERNEL_PROXY
1401 static void
1402 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
1403 {
1404         struct cfiscsi_session *cs;
1405
1406         cs = cfiscsi_session_new(&cfiscsi_softc, NULL);
1407         if (cs == NULL) {
1408                 CFISCSI_WARN("failed to create session");
1409                 return;
1410         }
1411
1412         icl_conn_handoff_sock(cs->cs_conn, so);
1413         cs->cs_initiator_sa = sa;
1414         cs->cs_portal_id = portal_id;
1415         cs->cs_handoff_in_progress = false;
1416         cs->cs_waiting_for_ctld = true;
1417         cv_signal(&cfiscsi_softc.accept_cv);
1418
1419         CFISCSI_SESSION_LOCK(cs);
1420         /*
1421          * Wake up the maintenance thread if we got scheduled for termination
1422          * somewhere between cfiscsi_session_new() and icl_conn_handoff_sock().
1423          */
1424         if (cs->cs_terminating)
1425                 cfiscsi_session_terminate(cs);
1426         CFISCSI_SESSION_UNLOCK(cs);
1427 }
1428 #endif
1429
1430 static void
1431 cfiscsi_online(void *arg)
1432 {
1433         struct cfiscsi_softc *softc;
1434         struct cfiscsi_target *ct;
1435         int online;
1436
1437         ct = (struct cfiscsi_target *)arg;
1438         softc = ct->ct_softc;
1439
1440         mtx_lock(&softc->lock);
1441         if (ct->ct_online) {
1442                 mtx_unlock(&softc->lock);
1443                 return;
1444         }
1445         ct->ct_online = 1;
1446         online = softc->online++;
1447         mtx_unlock(&softc->lock);
1448         if (online > 0)
1449                 return;
1450
1451 #ifdef ICL_KERNEL_PROXY
1452         if (softc->listener != NULL)
1453                 icl_listen_free(softc->listener);
1454         softc->listener = icl_listen_new(cfiscsi_accept);
1455 #endif
1456 }
1457
1458 static void
1459 cfiscsi_offline(void *arg)
1460 {
1461         struct cfiscsi_softc *softc;
1462         struct cfiscsi_target *ct;
1463         struct cfiscsi_session *cs;
1464         int error, online;
1465
1466         ct = (struct cfiscsi_target *)arg;
1467         softc = ct->ct_softc;
1468
1469         mtx_lock(&softc->lock);
1470         if (!ct->ct_online) {
1471                 mtx_unlock(&softc->lock);
1472                 return;
1473         }
1474         ct->ct_online = 0;
1475         online = --softc->online;
1476
1477         do {
1478                 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1479                         if (cs->cs_target == ct)
1480                                 cfiscsi_session_terminate(cs);
1481                 }
1482                 TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1483                         if (cs->cs_target == ct)
1484                                 break;
1485                 }
1486                 if (cs != NULL) {
1487                         error = cv_wait_sig(&softc->sessions_cv, &softc->lock);
1488                         if (error != 0) {
1489                                 CFISCSI_SESSION_DEBUG(cs,
1490                                     "cv_wait failed with error %d\n", error);
1491                                 break;
1492                         }
1493                 }
1494         } while (cs != NULL && ct->ct_online == 0);
1495         mtx_unlock(&softc->lock);
1496         if (online > 0)
1497                 return;
1498
1499 #ifdef ICL_KERNEL_PROXY
1500         icl_listen_free(softc->listener);
1501         softc->listener = NULL;
1502 #endif
1503 }
1504
1505 static int
1506 cfiscsi_info(void *arg, struct sbuf *sb)
1507 {
1508         struct cfiscsi_target *ct = (struct cfiscsi_target *)arg;
1509         int retval;
1510
1511         retval = sbuf_printf(sb, "\t<cfiscsi_state>%d</cfiscsi_state>\n",
1512             ct->ct_state);
1513         return (retval);
1514 }
1515
1516 static void
1517 cfiscsi_ioctl_handoff(struct ctl_iscsi *ci)
1518 {
1519         struct cfiscsi_softc *softc;
1520         struct cfiscsi_session *cs, *cs2;
1521         struct cfiscsi_target *ct;
1522         struct ctl_iscsi_handoff_params *cihp;
1523         int error;
1524
1525         cihp = (struct ctl_iscsi_handoff_params *)&(ci->data);
1526         softc = &cfiscsi_softc;
1527
1528         CFISCSI_DEBUG("new connection from %s (%s) to %s",
1529             cihp->initiator_name, cihp->initiator_addr,
1530             cihp->target_name);
1531
1532         ct = cfiscsi_target_find(softc, cihp->target_name,
1533             cihp->portal_group_tag);
1534         if (ct == NULL) {
1535                 ci->status = CTL_ISCSI_ERROR;
1536                 snprintf(ci->error_str, sizeof(ci->error_str),
1537                     "%s: target not found", __func__);
1538                 return;
1539         }
1540
1541 #ifdef ICL_KERNEL_PROXY
1542         if (cihp->socket > 0 && cihp->connection_id > 0) {
1543                 snprintf(ci->error_str, sizeof(ci->error_str),
1544                     "both socket and connection_id set");
1545                 ci->status = CTL_ISCSI_ERROR;
1546                 cfiscsi_target_release(ct);
1547                 return;
1548         }
1549         if (cihp->socket == 0) {
1550                 mtx_lock(&cfiscsi_softc.lock);
1551                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1552                         if (cs->cs_id == cihp->connection_id)
1553                                 break;
1554                 }
1555                 if (cs == NULL) {
1556                         mtx_unlock(&cfiscsi_softc.lock);
1557                         snprintf(ci->error_str, sizeof(ci->error_str),
1558                             "connection not found");
1559                         ci->status = CTL_ISCSI_ERROR;
1560                         cfiscsi_target_release(ct);
1561                         return;
1562                 }
1563                 mtx_unlock(&cfiscsi_softc.lock);
1564         } else {
1565 #endif
1566                 cs = cfiscsi_session_new(softc, cihp->offload);
1567                 if (cs == NULL) {
1568                         ci->status = CTL_ISCSI_ERROR;
1569                         snprintf(ci->error_str, sizeof(ci->error_str),
1570                             "%s: cfiscsi_session_new failed", __func__);
1571                         cfiscsi_target_release(ct);
1572                         return;
1573                 }
1574 #ifdef ICL_KERNEL_PROXY
1575         }
1576 #endif
1577
1578         /*
1579          * First PDU of Full Feature phase has the same CmdSN as the last
1580          * PDU from the Login Phase received from the initiator.  Thus,
1581          * the -1 below.
1582          */
1583         cs->cs_cmdsn = cihp->cmdsn;
1584         cs->cs_statsn = cihp->statsn;
1585         cs->cs_conn->ic_max_recv_data_segment_length =
1586             cihp->max_recv_data_segment_length;
1587         cs->cs_conn->ic_max_send_data_segment_length =
1588             cihp->max_send_data_segment_length;
1589         cs->cs_max_burst_length = cihp->max_burst_length;
1590         cs->cs_first_burst_length = cihp->first_burst_length;
1591         cs->cs_immediate_data = !!cihp->immediate_data;
1592         if (cihp->header_digest == CTL_ISCSI_DIGEST_CRC32C)
1593                 cs->cs_conn->ic_header_crc32c = true;
1594         if (cihp->data_digest == CTL_ISCSI_DIGEST_CRC32C)
1595                 cs->cs_conn->ic_data_crc32c = true;
1596
1597         strlcpy(cs->cs_initiator_name,
1598             cihp->initiator_name, sizeof(cs->cs_initiator_name));
1599         strlcpy(cs->cs_initiator_addr,
1600             cihp->initiator_addr, sizeof(cs->cs_initiator_addr));
1601         strlcpy(cs->cs_initiator_alias,
1602             cihp->initiator_alias, sizeof(cs->cs_initiator_alias));
1603         memcpy(cs->cs_initiator_isid,
1604             cihp->initiator_isid, sizeof(cs->cs_initiator_isid));
1605         snprintf(cs->cs_initiator_id, sizeof(cs->cs_initiator_id),
1606             "%s,i,0x%02x%02x%02x%02x%02x%02x", cs->cs_initiator_name,
1607             cihp->initiator_isid[0], cihp->initiator_isid[1],
1608             cihp->initiator_isid[2], cihp->initiator_isid[3],
1609             cihp->initiator_isid[4], cihp->initiator_isid[5]);
1610
1611         mtx_lock(&softc->lock);
1612         if (ct->ct_online == 0) {
1613                 mtx_unlock(&softc->lock);
1614                 CFISCSI_SESSION_LOCK(cs);
1615                 cs->cs_handoff_in_progress = false;
1616                 cfiscsi_session_terminate(cs);
1617                 CFISCSI_SESSION_UNLOCK(cs);
1618                 cfiscsi_target_release(ct);
1619                 ci->status = CTL_ISCSI_ERROR;
1620                 snprintf(ci->error_str, sizeof(ci->error_str),
1621                     "%s: port offline", __func__);
1622                 return;
1623         }
1624         cs->cs_target = ct;
1625         mtx_unlock(&softc->lock);
1626
1627 restart:
1628         if (!cs->cs_terminating) {
1629                 mtx_lock(&softc->lock);
1630                 TAILQ_FOREACH(cs2, &softc->sessions, cs_next) {
1631                         if (cs2 != cs && cs2->cs_tasks_aborted == false &&
1632                             cs->cs_target == cs2->cs_target &&
1633                             strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
1634                                 if (strcmp(cs->cs_initiator_addr,
1635                                     cs2->cs_initiator_addr) != 0) {
1636                                         CFISCSI_SESSION_WARN(cs2,
1637                                             "session reinstatement from "
1638                                             "different address %s",
1639                                             cs->cs_initiator_addr);
1640                                 } else {
1641                                         CFISCSI_SESSION_DEBUG(cs2,
1642                                             "session reinstatement");
1643                                 }
1644                                 cfiscsi_session_terminate(cs2);
1645                                 mtx_unlock(&softc->lock);
1646                                 pause("cfiscsi_reinstate", 1);
1647                                 goto restart;
1648                         }
1649                 }
1650                 mtx_unlock(&softc->lock);
1651         }
1652
1653         /*
1654          * Register initiator with CTL.
1655          */
1656         cfiscsi_session_register_initiator(cs);
1657
1658 #ifdef ICL_KERNEL_PROXY
1659         if (cihp->socket > 0) {
1660 #endif
1661                 error = icl_conn_handoff(cs->cs_conn, cihp->socket);
1662                 if (error != 0) {
1663                         CFISCSI_SESSION_LOCK(cs);
1664                         cs->cs_handoff_in_progress = false;
1665                         cfiscsi_session_terminate(cs);
1666                         CFISCSI_SESSION_UNLOCK(cs);
1667                         ci->status = CTL_ISCSI_ERROR;
1668                         snprintf(ci->error_str, sizeof(ci->error_str),
1669                             "%s: icl_conn_handoff failed with error %d",
1670                             __func__, error);
1671                         return;
1672                 }
1673 #ifdef ICL_KERNEL_PROXY
1674         }
1675 #endif
1676
1677 #ifdef ICL_KERNEL_PROXY
1678         cs->cs_login_phase = false;
1679
1680         /*
1681          * First PDU of the Full Feature phase has likely already arrived.
1682          * We have to pick it up and execute properly.
1683          */
1684         if (cs->cs_login_pdu != NULL) {
1685                 CFISCSI_SESSION_DEBUG(cs, "picking up first PDU");
1686                 cfiscsi_pdu_handle(cs->cs_login_pdu);
1687                 cs->cs_login_pdu = NULL;
1688         }
1689 #endif
1690
1691         CFISCSI_SESSION_LOCK(cs);
1692         cs->cs_handoff_in_progress = false;
1693
1694         /*
1695          * Wake up the maintenance thread if we got scheduled for termination.
1696          */
1697         if (cs->cs_terminating)
1698                 cfiscsi_session_terminate(cs);
1699         CFISCSI_SESSION_UNLOCK(cs);
1700
1701         ci->status = CTL_ISCSI_OK;
1702 }
1703
1704 static void
1705 cfiscsi_ioctl_list(struct ctl_iscsi *ci)
1706 {
1707         struct ctl_iscsi_list_params *cilp;
1708         struct cfiscsi_session *cs;
1709         struct cfiscsi_softc *softc;
1710         struct sbuf *sb;
1711         int error;
1712
1713         cilp = (struct ctl_iscsi_list_params *)&(ci->data);
1714         softc = &cfiscsi_softc;
1715
1716         sb = sbuf_new(NULL, NULL, cilp->alloc_len, SBUF_FIXEDLEN);
1717         if (sb == NULL) {
1718                 ci->status = CTL_ISCSI_ERROR;
1719                 snprintf(ci->error_str, sizeof(ci->error_str),
1720                     "Unable to allocate %d bytes for iSCSI session list",
1721                     cilp->alloc_len);
1722                 return;
1723         }
1724
1725         sbuf_printf(sb, "<ctlislist>\n");
1726         mtx_lock(&softc->lock);
1727         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1728                 if (cs->cs_target == NULL)
1729                         continue;
1730                 error = sbuf_printf(sb, "<connection id=\"%d\">"
1731                     "<initiator>%s</initiator>"
1732                     "<initiator_addr>%s</initiator_addr>"
1733                     "<initiator_alias>%s</initiator_alias>"
1734                     "<target>%s</target>"
1735                     "<target_alias>%s</target_alias>"
1736                     "<target_portal_group_tag>%u</target_portal_group_tag>"
1737                     "<header_digest>%s</header_digest>"
1738                     "<data_digest>%s</data_digest>"
1739                     "<max_recv_data_segment_length>%d</max_recv_data_segment_length>"
1740                     "<max_send_data_segment_length>%d</max_send_data_segment_length>"
1741                     "<max_burst_length>%d</max_burst_length>"
1742                     "<first_burst_length>%d</first_burst_length>"
1743                     "<immediate_data>%d</immediate_data>"
1744                     "<iser>%d</iser>"
1745                     "<offload>%s</offload>"
1746                     "</connection>\n",
1747                     cs->cs_id,
1748                     cs->cs_initiator_name, cs->cs_initiator_addr, cs->cs_initiator_alias,
1749                     cs->cs_target->ct_name, cs->cs_target->ct_alias,
1750                     cs->cs_target->ct_tag,
1751                     cs->cs_conn->ic_header_crc32c ? "CRC32C" : "None",
1752                     cs->cs_conn->ic_data_crc32c ? "CRC32C" : "None",
1753                     cs->cs_conn->ic_max_recv_data_segment_length,
1754                     cs->cs_conn->ic_max_send_data_segment_length,
1755                     cs->cs_max_burst_length,
1756                     cs->cs_first_burst_length,
1757                     cs->cs_immediate_data,
1758                     cs->cs_conn->ic_iser,
1759                     cs->cs_conn->ic_offload);
1760                 if (error != 0)
1761                         break;
1762         }
1763         mtx_unlock(&softc->lock);
1764         error = sbuf_printf(sb, "</ctlislist>\n");
1765         if (error != 0) {
1766                 sbuf_delete(sb);
1767                 ci->status = CTL_ISCSI_LIST_NEED_MORE_SPACE;
1768                 snprintf(ci->error_str, sizeof(ci->error_str),
1769                     "Out of space, %d bytes is too small", cilp->alloc_len);
1770                 return;
1771         }
1772         sbuf_finish(sb);
1773
1774         error = copyout(sbuf_data(sb), cilp->conn_xml, sbuf_len(sb) + 1);
1775         if (error != 0) {
1776                 sbuf_delete(sb);
1777                 snprintf(ci->error_str, sizeof(ci->error_str),
1778                     "copyout failed with error %d", error);
1779                 ci->status = CTL_ISCSI_ERROR;
1780                 return;
1781         }
1782         cilp->fill_len = sbuf_len(sb) + 1;
1783         ci->status = CTL_ISCSI_OK;
1784         sbuf_delete(sb);
1785 }
1786
1787 static void
1788 cfiscsi_ioctl_logout(struct ctl_iscsi *ci)
1789 {
1790         struct icl_pdu *response;
1791         struct iscsi_bhs_asynchronous_message *bhsam;
1792         struct ctl_iscsi_logout_params *cilp;
1793         struct cfiscsi_session *cs;
1794         struct cfiscsi_softc *softc;
1795         int found = 0;
1796
1797         cilp = (struct ctl_iscsi_logout_params *)&(ci->data);
1798         softc = &cfiscsi_softc;
1799
1800         mtx_lock(&softc->lock);
1801         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1802                 if (cilp->all == 0 && cs->cs_id != cilp->connection_id &&
1803                     strcmp(cs->cs_initiator_name, cilp->initiator_name) != 0 &&
1804                     strcmp(cs->cs_initiator_addr, cilp->initiator_addr) != 0)
1805                         continue;
1806
1807                 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1808                 if (response == NULL) {
1809                         ci->status = CTL_ISCSI_ERROR;
1810                         snprintf(ci->error_str, sizeof(ci->error_str),
1811                             "Unable to allocate memory");
1812                         mtx_unlock(&softc->lock);
1813                         return;
1814                 }
1815                 bhsam =
1816                     (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1817                 bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1818                 bhsam->bhsam_flags = 0x80;
1819                 bhsam->bhsam_async_event = BHSAM_EVENT_TARGET_REQUESTS_LOGOUT;
1820                 bhsam->bhsam_parameter3 = htons(10);
1821                 cfiscsi_pdu_queue(response);
1822                 found++;
1823         }
1824         mtx_unlock(&softc->lock);
1825
1826         if (found == 0) {
1827                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1828                 snprintf(ci->error_str, sizeof(ci->error_str),
1829                     "No matching connections found");
1830                 return;
1831         }
1832
1833         ci->status = CTL_ISCSI_OK;
1834 }
1835
1836 static void
1837 cfiscsi_ioctl_terminate(struct ctl_iscsi *ci)
1838 {
1839         struct icl_pdu *response;
1840         struct iscsi_bhs_asynchronous_message *bhsam;
1841         struct ctl_iscsi_terminate_params *citp;
1842         struct cfiscsi_session *cs;
1843         struct cfiscsi_softc *softc;
1844         int found = 0;
1845
1846         citp = (struct ctl_iscsi_terminate_params *)&(ci->data);
1847         softc = &cfiscsi_softc;
1848
1849         mtx_lock(&softc->lock);
1850         TAILQ_FOREACH(cs, &softc->sessions, cs_next) {
1851                 if (citp->all == 0 && cs->cs_id != citp->connection_id &&
1852                     strcmp(cs->cs_initiator_name, citp->initiator_name) != 0 &&
1853                     strcmp(cs->cs_initiator_addr, citp->initiator_addr) != 0)
1854                         continue;
1855
1856                 response = icl_pdu_new(cs->cs_conn, M_NOWAIT);
1857                 if (response == NULL) {
1858                         /*
1859                          * Oh well.  Just terminate the connection.
1860                          */
1861                 } else {
1862                         bhsam = (struct iscsi_bhs_asynchronous_message *)
1863                             response->ip_bhs;
1864                         bhsam->bhsam_opcode = ISCSI_BHS_OPCODE_ASYNC_MESSAGE;
1865                         bhsam->bhsam_flags = 0x80;
1866                         bhsam->bhsam_0xffffffff = 0xffffffff;
1867                         bhsam->bhsam_async_event =
1868                             BHSAM_EVENT_TARGET_TERMINATES_SESSION;
1869                         cfiscsi_pdu_queue(response);
1870                 }
1871                 cfiscsi_session_terminate(cs);
1872                 found++;
1873         }
1874         mtx_unlock(&softc->lock);
1875
1876         if (found == 0) {
1877                 ci->status = CTL_ISCSI_SESSION_NOT_FOUND;
1878                 snprintf(ci->error_str, sizeof(ci->error_str),
1879                     "No matching connections found");
1880                 return;
1881         }
1882
1883         ci->status = CTL_ISCSI_OK;
1884 }
1885
1886 static void
1887 cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
1888 {
1889         struct ctl_iscsi_limits_params *cilp;
1890         struct icl_drv_limits idl;
1891         int error;
1892
1893         cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
1894
1895         error = icl_limits(cilp->offload, false, cilp->socket, &idl);
1896         if (error != 0) {
1897                 ci->status = CTL_ISCSI_ERROR;
1898                 snprintf(ci->error_str, sizeof(ci->error_str),
1899                         "%s: icl_limits failed with error %d",
1900                         __func__, error);
1901                 return;
1902         }
1903
1904         cilp->max_recv_data_segment_length =
1905             idl.idl_max_recv_data_segment_length;
1906         cilp->max_send_data_segment_length =
1907             idl.idl_max_send_data_segment_length;
1908         cilp->max_burst_length = idl.idl_max_burst_length;
1909         cilp->first_burst_length = idl.idl_first_burst_length;
1910
1911         ci->status = CTL_ISCSI_OK;
1912 }
1913
1914 #ifdef ICL_KERNEL_PROXY
1915 static void
1916 cfiscsi_ioctl_listen(struct ctl_iscsi *ci)
1917 {
1918         struct ctl_iscsi_listen_params *cilp;
1919         struct sockaddr *sa;
1920         int error;
1921
1922         cilp = (struct ctl_iscsi_listen_params *)&(ci->data);
1923
1924         if (cfiscsi_softc.listener == NULL) {
1925                 CFISCSI_DEBUG("no listener");
1926                 snprintf(ci->error_str, sizeof(ci->error_str), "no listener");
1927                 ci->status = CTL_ISCSI_ERROR;
1928                 return;
1929         }
1930
1931         error = getsockaddr(&sa, (void *)cilp->addr, cilp->addrlen);
1932         if (error != 0) {
1933                 CFISCSI_DEBUG("getsockaddr, error %d", error);
1934                 snprintf(ci->error_str, sizeof(ci->error_str), "getsockaddr failed");
1935                 ci->status = CTL_ISCSI_ERROR;
1936                 return;
1937         }
1938
1939         error = icl_listen_add(cfiscsi_softc.listener, cilp->iser, cilp->domain,
1940             cilp->socktype, cilp->protocol, sa, cilp->portal_id);
1941         if (error != 0) {
1942                 free(sa, M_SONAME);
1943                 CFISCSI_DEBUG("icl_listen_add, error %d", error);
1944                 snprintf(ci->error_str, sizeof(ci->error_str),
1945                     "icl_listen_add failed, error %d", error);
1946                 ci->status = CTL_ISCSI_ERROR;
1947                 return;
1948         }
1949
1950         ci->status = CTL_ISCSI_OK;
1951 }
1952
1953 static void
1954 cfiscsi_ioctl_accept(struct ctl_iscsi *ci)
1955 {
1956         struct ctl_iscsi_accept_params *ciap;
1957         struct cfiscsi_session *cs;
1958         int error;
1959
1960         ciap = (struct ctl_iscsi_accept_params *)&(ci->data);
1961
1962         mtx_lock(&cfiscsi_softc.lock);
1963         for (;;) {
1964                 TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
1965                         if (cs->cs_waiting_for_ctld)
1966                                 break;
1967                 }
1968                 if (cs != NULL)
1969                         break;
1970                 error = cv_wait_sig(&cfiscsi_softc.accept_cv, &cfiscsi_softc.lock);
1971                 if (error != 0) {
1972                         mtx_unlock(&cfiscsi_softc.lock);
1973                         snprintf(ci->error_str, sizeof(ci->error_str), "interrupted");
1974                         ci->status = CTL_ISCSI_ERROR;
1975                         return;
1976                 }
1977         }
1978         mtx_unlock(&cfiscsi_softc.lock);
1979
1980         cs->cs_waiting_for_ctld = false;
1981         cs->cs_login_phase = true;
1982
1983         ciap->connection_id = cs->cs_id;
1984         ciap->portal_id = cs->cs_portal_id;
1985         ciap->initiator_addrlen = cs->cs_initiator_sa->sa_len;
1986         error = copyout(cs->cs_initiator_sa, ciap->initiator_addr,
1987             cs->cs_initiator_sa->sa_len);
1988         if (error != 0) {
1989                 snprintf(ci->error_str, sizeof(ci->error_str),
1990                     "copyout failed with error %d", error);
1991                 ci->status = CTL_ISCSI_ERROR;
1992                 return;
1993         }
1994
1995         ci->status = CTL_ISCSI_OK;
1996 }
1997
1998 static void
1999 cfiscsi_ioctl_send(struct ctl_iscsi *ci)
2000 {
2001         struct ctl_iscsi_send_params *cisp;
2002         struct cfiscsi_session *cs;
2003         struct icl_pdu *ip;
2004         size_t datalen;
2005         void *data;
2006         int error;
2007
2008         cisp = (struct ctl_iscsi_send_params *)&(ci->data);
2009
2010         mtx_lock(&cfiscsi_softc.lock);
2011         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2012                 if (cs->cs_id == cisp->connection_id)
2013                         break;
2014         }
2015         if (cs == NULL) {
2016                 mtx_unlock(&cfiscsi_softc.lock);
2017                 snprintf(ci->error_str, sizeof(ci->error_str), "connection not found");
2018                 ci->status = CTL_ISCSI_ERROR;
2019                 return;
2020         }
2021         mtx_unlock(&cfiscsi_softc.lock);
2022
2023 #if 0
2024         if (cs->cs_login_phase == false)
2025                 return (EBUSY);
2026 #endif
2027
2028         if (cs->cs_terminating) {
2029                 snprintf(ci->error_str, sizeof(ci->error_str), "connection is terminating");
2030                 ci->status = CTL_ISCSI_ERROR;
2031                 return;
2032         }
2033
2034         datalen = cisp->data_segment_len;
2035         /*
2036          * XXX
2037          */
2038         //if (datalen > CFISCSI_MAX_DATA_SEGMENT_LENGTH) {
2039         if (datalen > 65535) {
2040                 snprintf(ci->error_str, sizeof(ci->error_str), "data segment too big");
2041                 ci->status = CTL_ISCSI_ERROR;
2042                 return;
2043         }
2044         if (datalen > 0) {
2045                 data = malloc(datalen, M_CFISCSI, M_WAITOK);
2046                 error = copyin(cisp->data_segment, data, datalen);
2047                 if (error != 0) {
2048                         free(data, M_CFISCSI);
2049                         snprintf(ci->error_str, sizeof(ci->error_str), "copyin error %d", error);
2050                         ci->status = CTL_ISCSI_ERROR;
2051                         return;
2052                 }
2053         }
2054
2055         ip = icl_pdu_new(cs->cs_conn, M_WAITOK);
2056         memcpy(ip->ip_bhs, cisp->bhs, sizeof(*ip->ip_bhs));
2057         if (datalen > 0) {
2058                 icl_pdu_append_data(ip, data, datalen, M_WAITOK);
2059                 free(data, M_CFISCSI);
2060         }
2061         CFISCSI_SESSION_LOCK(cs);
2062         icl_pdu_queue(ip);
2063         CFISCSI_SESSION_UNLOCK(cs);
2064         ci->status = CTL_ISCSI_OK;
2065 }
2066
2067 static void
2068 cfiscsi_ioctl_receive(struct ctl_iscsi *ci)
2069 {
2070         struct ctl_iscsi_receive_params *cirp;
2071         struct cfiscsi_session *cs;
2072         struct icl_pdu *ip;
2073         void *data;
2074         int error;
2075
2076         cirp = (struct ctl_iscsi_receive_params *)&(ci->data);
2077
2078         mtx_lock(&cfiscsi_softc.lock);
2079         TAILQ_FOREACH(cs, &cfiscsi_softc.sessions, cs_next) {
2080                 if (cs->cs_id == cirp->connection_id)
2081                         break;
2082         }
2083         if (cs == NULL) {
2084                 mtx_unlock(&cfiscsi_softc.lock);
2085                 snprintf(ci->error_str, sizeof(ci->error_str),
2086                     "connection not found");
2087                 ci->status = CTL_ISCSI_ERROR;
2088                 return;
2089         }
2090         mtx_unlock(&cfiscsi_softc.lock);
2091
2092 #if 0
2093         if (is->is_login_phase == false)
2094                 return (EBUSY);
2095 #endif
2096
2097         CFISCSI_SESSION_LOCK(cs);
2098         while (cs->cs_login_pdu == NULL && cs->cs_terminating == false) {
2099                 error = cv_wait_sig(&cs->cs_login_cv, &cs->cs_lock);
2100                 if (error != 0) {
2101                         CFISCSI_SESSION_UNLOCK(cs);
2102                         snprintf(ci->error_str, sizeof(ci->error_str),
2103                             "interrupted by signal");
2104                         ci->status = CTL_ISCSI_ERROR;
2105                         return;
2106                 }
2107         }
2108
2109         if (cs->cs_terminating) {
2110                 CFISCSI_SESSION_UNLOCK(cs);
2111                 snprintf(ci->error_str, sizeof(ci->error_str),
2112                     "connection terminating");
2113                 ci->status = CTL_ISCSI_ERROR;
2114                 return;
2115         }
2116         ip = cs->cs_login_pdu;
2117         cs->cs_login_pdu = NULL;
2118         CFISCSI_SESSION_UNLOCK(cs);
2119
2120         if (ip->ip_data_len > cirp->data_segment_len) {
2121                 icl_pdu_free(ip);
2122                 snprintf(ci->error_str, sizeof(ci->error_str),
2123                     "data segment too big");
2124                 ci->status = CTL_ISCSI_ERROR;
2125                 return;
2126         }
2127
2128         copyout(ip->ip_bhs, cirp->bhs, sizeof(*ip->ip_bhs));
2129         if (ip->ip_data_len > 0) {
2130                 data = malloc(ip->ip_data_len, M_CFISCSI, M_WAITOK);
2131                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
2132                 copyout(data, cirp->data_segment, ip->ip_data_len);
2133                 free(data, M_CFISCSI);
2134         }
2135
2136         icl_pdu_free(ip);
2137         ci->status = CTL_ISCSI_OK;
2138 }
2139
2140 #endif /* !ICL_KERNEL_PROXY */
2141
2142 static void
2143 cfiscsi_ioctl_port_create(struct ctl_req *req)
2144 {
2145         struct cfiscsi_target *ct;
2146         struct ctl_port *port;
2147         const char *target, *alias, *val;
2148         struct scsi_vpd_id_descriptor *desc;
2149         int retval, len, idlen;
2150         uint16_t tag;
2151
2152         target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2153         alias = dnvlist_get_string(req->args_nvl, "cfiscsi_target_alias", NULL);
2154         val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2155             NULL);
2156
2157         if (target == NULL || val == NULL) {
2158                 req->status = CTL_LUN_ERROR;
2159                 snprintf(req->error_str, sizeof(req->error_str),
2160                     "Missing required argument");
2161                 return;
2162         }
2163
2164         tag = strtoul(val, NULL, 0);
2165         ct = cfiscsi_target_find_or_create(&cfiscsi_softc, target, alias, tag);
2166         if (ct == NULL) {
2167                 req->status = CTL_LUN_ERROR;
2168                 snprintf(req->error_str, sizeof(req->error_str),
2169                     "failed to create target \"%s\"", target);
2170                 return;
2171         }
2172         if (ct->ct_state == CFISCSI_TARGET_STATE_ACTIVE) {
2173                 req->status = CTL_LUN_ERROR;
2174                 snprintf(req->error_str, sizeof(req->error_str),
2175                     "target \"%s\" for portal group tag %u already exists",
2176                     target, tag);
2177                 cfiscsi_target_release(ct);
2178                 return;
2179         }
2180         port = &ct->ct_port;
2181         // WAT
2182         if (ct->ct_state == CFISCSI_TARGET_STATE_DYING)
2183                 goto done;
2184
2185         port->frontend = &cfiscsi_frontend;
2186         port->port_type = CTL_PORT_ISCSI;
2187         /* XXX KDM what should the real number be here? */
2188         port->num_requested_ctl_io = 4096;
2189         port->port_name = "iscsi";
2190         port->physical_port = (int)tag;
2191         port->virtual_port = ct->ct_target_id;
2192         port->port_online = cfiscsi_online;
2193         port->port_offline = cfiscsi_offline;
2194         port->port_info = cfiscsi_info;
2195         port->onoff_arg = ct;
2196         port->fe_datamove = cfiscsi_datamove;
2197         port->fe_done = cfiscsi_done;
2198         port->targ_port = -1;
2199         port->options = nvlist_clone(req->args_nvl);
2200
2201         /* Generate Port ID. */
2202         idlen = strlen(target) + strlen(",t,0x0001") + 1;
2203         idlen = roundup2(idlen, 4);
2204         len = sizeof(struct scsi_vpd_device_id) + idlen;
2205         port->port_devid = malloc(sizeof(struct ctl_devid) + len,
2206             M_CTL, M_WAITOK | M_ZERO);
2207         port->port_devid->len = len;
2208         desc = (struct scsi_vpd_id_descriptor *)port->port_devid->data;
2209         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2210         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_PORT |
2211             SVPD_ID_TYPE_SCSI_NAME;
2212         desc->length = idlen;
2213         snprintf(desc->identifier, idlen, "%s,t,0x%4.4x", target, tag);
2214
2215         /* Generate Target ID. */
2216         idlen = strlen(target) + 1;
2217         idlen = roundup2(idlen, 4);
2218         len = sizeof(struct scsi_vpd_device_id) + idlen;
2219         port->target_devid = malloc(sizeof(struct ctl_devid) + len,
2220             M_CTL, M_WAITOK | M_ZERO);
2221         port->target_devid->len = len;
2222         desc = (struct scsi_vpd_id_descriptor *)port->target_devid->data;
2223         desc->proto_codeset = (SCSI_PROTO_ISCSI << 4) | SVPD_ID_CODESET_UTF8;
2224         desc->id_type = SVPD_ID_PIV | SVPD_ID_ASSOC_TARGET |
2225             SVPD_ID_TYPE_SCSI_NAME;
2226         desc->length = idlen;
2227         strlcpy(desc->identifier, target, idlen);
2228
2229         retval = ctl_port_register(port);
2230         if (retval != 0) {
2231                 free(port->port_devid, M_CFISCSI);
2232                 free(port->target_devid, M_CFISCSI);
2233                 cfiscsi_target_release(ct);
2234                 req->status = CTL_LUN_ERROR;
2235                 snprintf(req->error_str, sizeof(req->error_str),
2236                     "ctl_port_register() failed with error %d", retval);
2237                 return;
2238         }
2239 done:
2240         ct->ct_state = CFISCSI_TARGET_STATE_ACTIVE;
2241         req->status = CTL_LUN_OK;
2242         req->result_nvl = nvlist_create(0);
2243         nvlist_add_number(req->result_nvl, "port_id", port->targ_port);
2244 }
2245
2246 static void
2247 cfiscsi_ioctl_port_remove(struct ctl_req *req)
2248 {
2249         struct cfiscsi_target *ct;
2250         const char *target, *val;
2251         uint16_t tag;
2252
2253         target = dnvlist_get_string(req->args_nvl, "cfiscsi_target", NULL);
2254         val = dnvlist_get_string(req->args_nvl, "cfiscsi_portal_group_tag",
2255             NULL);
2256
2257         if (target == NULL || val == NULL) {
2258                 req->status = CTL_LUN_ERROR;
2259                 snprintf(req->error_str, sizeof(req->error_str),
2260                     "Missing required argument");
2261                 return;
2262         }
2263
2264         tag = strtoul(val, NULL, 0);
2265         ct = cfiscsi_target_find(&cfiscsi_softc, target, tag);
2266         if (ct == NULL) {
2267                 req->status = CTL_LUN_ERROR;
2268                 snprintf(req->error_str, sizeof(req->error_str),
2269                     "can't find target \"%s\"", target);
2270                 return;
2271         }
2272
2273         ct->ct_state = CFISCSI_TARGET_STATE_DYING;
2274         ctl_port_offline(&ct->ct_port);
2275         cfiscsi_target_release(ct);
2276         cfiscsi_target_release(ct);
2277         req->status = CTL_LUN_OK;
2278 }
2279
2280 static int
2281 cfiscsi_ioctl(struct cdev *dev,
2282     u_long cmd, caddr_t addr, int flag, struct thread *td)
2283 {
2284         struct ctl_iscsi *ci;
2285         struct ctl_req *req;
2286
2287         if (cmd == CTL_PORT_REQ) {
2288                 req = (struct ctl_req *)addr;
2289                 switch (req->reqtype) {
2290                 case CTL_REQ_CREATE:
2291                         cfiscsi_ioctl_port_create(req);
2292                         break;
2293                 case CTL_REQ_REMOVE:
2294                         cfiscsi_ioctl_port_remove(req);
2295                         break;
2296                 default:
2297                         req->status = CTL_LUN_ERROR;
2298                         snprintf(req->error_str, sizeof(req->error_str),
2299                             "Unsupported request type %d", req->reqtype);
2300                 }
2301                 return (0);
2302         }
2303
2304         if (cmd != CTL_ISCSI)
2305                 return (ENOTTY);
2306
2307         ci = (struct ctl_iscsi *)addr;
2308         switch (ci->type) {
2309         case CTL_ISCSI_HANDOFF:
2310                 cfiscsi_ioctl_handoff(ci);
2311                 break;
2312         case CTL_ISCSI_LIST:
2313                 cfiscsi_ioctl_list(ci);
2314                 break;
2315         case CTL_ISCSI_LOGOUT:
2316                 cfiscsi_ioctl_logout(ci);
2317                 break;
2318         case CTL_ISCSI_TERMINATE:
2319                 cfiscsi_ioctl_terminate(ci);
2320                 break;
2321         case CTL_ISCSI_LIMITS:
2322                 cfiscsi_ioctl_limits(ci);
2323                 break;
2324 #ifdef ICL_KERNEL_PROXY
2325         case CTL_ISCSI_LISTEN:
2326                 cfiscsi_ioctl_listen(ci);
2327                 break;
2328         case CTL_ISCSI_ACCEPT:
2329                 cfiscsi_ioctl_accept(ci);
2330                 break;
2331         case CTL_ISCSI_SEND:
2332                 cfiscsi_ioctl_send(ci);
2333                 break;
2334         case CTL_ISCSI_RECEIVE:
2335                 cfiscsi_ioctl_receive(ci);
2336                 break;
2337 #else
2338         case CTL_ISCSI_LISTEN:
2339         case CTL_ISCSI_ACCEPT:
2340         case CTL_ISCSI_SEND:
2341         case CTL_ISCSI_RECEIVE:
2342                 ci->status = CTL_ISCSI_ERROR;
2343                 snprintf(ci->error_str, sizeof(ci->error_str),
2344                     "%s: CTL compiled without ICL_KERNEL_PROXY",
2345                     __func__);
2346                 break;
2347 #endif /* !ICL_KERNEL_PROXY */
2348         default:
2349                 ci->status = CTL_ISCSI_ERROR;
2350                 snprintf(ci->error_str, sizeof(ci->error_str),
2351                     "%s: invalid iSCSI request type %d", __func__, ci->type);
2352                 break;
2353         }
2354
2355         return (0);
2356 }
2357
2358 static void
2359 cfiscsi_target_hold(struct cfiscsi_target *ct)
2360 {
2361
2362         refcount_acquire(&ct->ct_refcount);
2363 }
2364
2365 static void
2366 cfiscsi_target_release(struct cfiscsi_target *ct)
2367 {
2368         struct cfiscsi_softc *softc;
2369
2370         softc = ct->ct_softc;
2371         mtx_lock(&softc->lock);
2372         if (refcount_release(&ct->ct_refcount)) {
2373                 TAILQ_REMOVE(&softc->targets, ct, ct_next);
2374                 mtx_unlock(&softc->lock);
2375                 if (ct->ct_state != CFISCSI_TARGET_STATE_INVALID) {
2376                         ct->ct_state = CFISCSI_TARGET_STATE_INVALID;
2377                         if (ctl_port_deregister(&ct->ct_port) != 0)
2378                                 printf("%s: ctl_port_deregister() failed\n",
2379                                     __func__);
2380                 }
2381                 free(ct, M_CFISCSI);
2382
2383                 return;
2384         }
2385         mtx_unlock(&softc->lock);
2386 }
2387
2388 static struct cfiscsi_target *
2389 cfiscsi_target_find(struct cfiscsi_softc *softc, const char *name, uint16_t tag)
2390 {
2391         struct cfiscsi_target *ct;
2392
2393         mtx_lock(&softc->lock);
2394         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2395                 if (ct->ct_tag != tag ||
2396                     strcmp(name, ct->ct_name) != 0 ||
2397                     ct->ct_state != CFISCSI_TARGET_STATE_ACTIVE)
2398                         continue;
2399                 cfiscsi_target_hold(ct);
2400                 mtx_unlock(&softc->lock);
2401                 return (ct);
2402         }
2403         mtx_unlock(&softc->lock);
2404
2405         return (NULL);
2406 }
2407
2408 static struct cfiscsi_target *
2409 cfiscsi_target_find_or_create(struct cfiscsi_softc *softc, const char *name,
2410     const char *alias, uint16_t tag)
2411 {
2412         struct cfiscsi_target *ct, *newct;
2413
2414         if (name[0] == '\0' || strlen(name) >= CTL_ISCSI_NAME_LEN)
2415                 return (NULL);
2416
2417         newct = malloc(sizeof(*newct), M_CFISCSI, M_WAITOK | M_ZERO);
2418
2419         mtx_lock(&softc->lock);
2420         TAILQ_FOREACH(ct, &softc->targets, ct_next) {
2421                 if (ct->ct_tag != tag ||
2422                     strcmp(name, ct->ct_name) != 0 ||
2423                     ct->ct_state == CFISCSI_TARGET_STATE_INVALID)
2424                         continue;
2425                 cfiscsi_target_hold(ct);
2426                 mtx_unlock(&softc->lock);
2427                 free(newct, M_CFISCSI);
2428                 return (ct);
2429         }
2430
2431         strlcpy(newct->ct_name, name, sizeof(newct->ct_name));
2432         if (alias != NULL)
2433                 strlcpy(newct->ct_alias, alias, sizeof(newct->ct_alias));
2434         newct->ct_tag = tag;
2435         refcount_init(&newct->ct_refcount, 1);
2436         newct->ct_softc = softc;
2437         if (TAILQ_EMPTY(&softc->targets))
2438                 softc->last_target_id = 0;
2439         newct->ct_target_id = ++softc->last_target_id;
2440         TAILQ_INSERT_TAIL(&softc->targets, newct, ct_next);
2441         mtx_unlock(&softc->lock);
2442
2443         return (newct);
2444 }
2445
2446 static void
2447 cfiscsi_pdu_done(struct icl_pdu *ip, int error)
2448 {
2449
2450         if (error != 0)
2451                 ; // XXX: Do something on error?
2452         ((ctl_ref)ip->ip_prv0)(ip->ip_prv1, -1);
2453 }
2454
2455 static void
2456 cfiscsi_datamove_in(union ctl_io *io)
2457 {
2458         struct cfiscsi_session *cs;
2459         struct icl_pdu *request, *response;
2460         const struct iscsi_bhs_scsi_command *bhssc;
2461         struct iscsi_bhs_data_in *bhsdi;
2462         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2463         size_t len, expected_len, sg_len, buffer_offset;
2464         size_t max_send_data_segment_length;
2465         const char *sg_addr;
2466         icl_pdu_cb cb;
2467         int ctl_sg_count, error, i;
2468
2469         request = PRIV_REQUEST(io);
2470         cs = PDU_SESSION(request);
2471
2472         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2473         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2474             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2475             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2476
2477         if (io->scsiio.kern_sg_entries > 0) {
2478                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2479                 ctl_sg_count = io->scsiio.kern_sg_entries;
2480         } else {
2481                 ctl_sglist = &ctl_sg_entry;
2482                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2483                 ctl_sglist->len = io->scsiio.kern_data_len;
2484                 ctl_sg_count = 1;
2485         }
2486
2487         /*
2488          * This is the offset within the current SCSI command; for the first
2489          * call to cfiscsi_datamove() it will be 0, and for subsequent ones
2490          * it will be the sum of lengths of previous ones.
2491          */
2492         buffer_offset = io->scsiio.kern_rel_offset;
2493
2494         /*
2495          * This is the transfer length expected by the initiator.  It can be
2496          * different from the amount of data from the SCSI point of view.
2497          */
2498         expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2499
2500         /*
2501          * If the transfer is outside of expected length -- we are done.
2502          */
2503         if (buffer_offset >= expected_len) {
2504 #if 0
2505                 CFISCSI_SESSION_DEBUG(cs, "buffer_offset = %zd, "
2506                     "already sent the expected len", buffer_offset);
2507 #endif
2508                 ctl_datamove_done(io, true);
2509                 return;
2510         }
2511
2512         if (io->scsiio.kern_data_ref != NULL)
2513                 cb = cfiscsi_pdu_done;
2514         else
2515                 cb = NULL;
2516
2517         i = 0;
2518         sg_addr = NULL;
2519         sg_len = 0;
2520         response = NULL;
2521         bhsdi = NULL;
2522         if (cs->cs_conn->ic_hw_isomax != 0)
2523                 max_send_data_segment_length = cs->cs_conn->ic_hw_isomax;
2524         else
2525                 max_send_data_segment_length =
2526                     cs->cs_conn->ic_max_send_data_segment_length;
2527         for (;;) {
2528                 if (response == NULL) {
2529                         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2530                         if (response == NULL) {
2531                                 CFISCSI_SESSION_WARN(cs, "failed to "
2532                                     "allocate memory; dropping connection");
2533                                 ctl_set_busy(&io->scsiio);
2534                                 ctl_datamove_done(io, true);
2535                                 cfiscsi_session_terminate(cs);
2536                                 return;
2537                         }
2538                         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
2539                         bhsdi->bhsdi_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_IN;
2540                         bhsdi->bhsdi_initiator_task_tag =
2541                             bhssc->bhssc_initiator_task_tag;
2542                         bhsdi->bhsdi_target_transfer_tag = 0xffffffff;
2543                         bhsdi->bhsdi_datasn = htonl(PRIV_EXPDATASN(io));
2544                         bhsdi->bhsdi_buffer_offset = htonl(buffer_offset);
2545                 }
2546
2547                 KASSERT(i < ctl_sg_count, ("i >= ctl_sg_count"));
2548                 if (sg_len == 0) {
2549                         sg_addr = ctl_sglist[i].addr;
2550                         sg_len = ctl_sglist[i].len;
2551                         KASSERT(sg_len > 0, ("sg_len <= 0"));
2552                 }
2553
2554                 len = sg_len;
2555
2556                 /*
2557                  * Truncate to maximum data segment length.
2558                  */
2559                 KASSERT(response->ip_data_len < max_send_data_segment_length,
2560                     ("ip_data_len %zd >= max_send_data_segment_length %zd",
2561                     response->ip_data_len, max_send_data_segment_length));
2562                 if (response->ip_data_len + len > max_send_data_segment_length) {
2563                         len = max_send_data_segment_length - response->ip_data_len;
2564                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2565                             len, sg_len));
2566                 }
2567
2568                 /*
2569                  * Truncate to expected data transfer length.
2570                  */
2571                 KASSERT(buffer_offset + response->ip_data_len < expected_len,
2572                     ("buffer_offset %zd + ip_data_len %zd >= expected_len %zd",
2573                     buffer_offset, response->ip_data_len, expected_len));
2574                 if (buffer_offset + response->ip_data_len + len > expected_len) {
2575                         CFISCSI_SESSION_DEBUG(cs, "truncating from %zd "
2576                             "to expected data transfer length %zd",
2577                             buffer_offset + response->ip_data_len + len, expected_len);
2578                         len = expected_len - (buffer_offset + response->ip_data_len);
2579                         KASSERT(len <= sg_len, ("len %zd > sg_len %zd",
2580                             len, sg_len));
2581                 }
2582
2583                 error = icl_pdu_append_data(response, sg_addr, len,
2584                     M_NOWAIT | (cb ? ICL_NOCOPY : 0));
2585                 if (error != 0) {
2586                         CFISCSI_SESSION_WARN(cs, "failed to "
2587                             "allocate memory; dropping connection");
2588                         icl_pdu_free(response);
2589                         ctl_set_busy(&io->scsiio);
2590                         ctl_datamove_done(io, true);
2591                         cfiscsi_session_terminate(cs);
2592                         return;
2593                 }
2594                 sg_addr += len;
2595                 sg_len -= len;
2596                 io->scsiio.kern_data_resid -= len;
2597
2598                 KASSERT(buffer_offset + response->ip_data_len <= expected_len,
2599                     ("buffer_offset %zd + ip_data_len %zd > expected_len %zd",
2600                     buffer_offset, response->ip_data_len, expected_len));
2601                 if (buffer_offset + response->ip_data_len == expected_len) {
2602                         /*
2603                          * Already have the amount of data the initiator wanted.
2604                          */
2605                         break;
2606                 }
2607
2608                 if (sg_len == 0) {
2609                         /*
2610                          * End of scatter-gather segment;
2611                          * proceed to the next one...
2612                          */
2613                         if (i == ctl_sg_count - 1) {
2614                                 /*
2615                                  * ... unless this was the last one.
2616                                  */
2617                                 break;
2618                         }
2619                         i++;
2620                 }
2621
2622                 if (response->ip_data_len == max_send_data_segment_length) {
2623                         /*
2624                          * Can't stuff more data into the current PDU;
2625                          * queue it.  Note that's not enough to check
2626                          * for kern_data_resid == 0 instead; there
2627                          * may be several Data-In PDUs for the final
2628                          * call to cfiscsi_datamove(), and we want
2629                          * to set the F flag only on the last of them.
2630                          */
2631                         buffer_offset += response->ip_data_len;
2632                         if (buffer_offset == io->scsiio.kern_total_len ||
2633                             buffer_offset == expected_len) {
2634                                 buffer_offset -= response->ip_data_len;
2635                                 break;
2636                         }
2637                         PRIV_EXPDATASN(io) += howmany(response->ip_data_len,
2638                             cs->cs_conn->ic_max_send_data_segment_length);
2639                         if (cb != NULL) {
2640                                 response->ip_prv0 = io->scsiio.kern_data_ref;
2641                                 response->ip_prv1 = io->scsiio.kern_data_arg;
2642                                 io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2643                         }
2644                         cfiscsi_pdu_queue_cb(response, cb);
2645                         response = NULL;
2646                         bhsdi = NULL;
2647                 }
2648         }
2649         if (response != NULL) {
2650                 buffer_offset += response->ip_data_len;
2651                 if (buffer_offset == io->scsiio.kern_total_len ||
2652                     buffer_offset == expected_len) {
2653                         bhsdi->bhsdi_flags |= BHSDI_FLAGS_F;
2654                         if (io->io_hdr.status == CTL_SUCCESS) {
2655                                 bhsdi->bhsdi_flags |= BHSDI_FLAGS_S;
2656                                 if (io->scsiio.kern_total_len <
2657                                     ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2658                                         bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2659                                         bhsdi->bhsdi_residual_count =
2660                                             htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2661                                             io->scsiio.kern_total_len);
2662                                 } else if (io->scsiio.kern_total_len >
2663                                     ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2664                                         bhsdi->bhsdi_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2665                                         bhsdi->bhsdi_residual_count =
2666                                             htonl(io->scsiio.kern_total_len -
2667                                             ntohl(bhssc->bhssc_expected_data_transfer_length));
2668                                 }
2669                                 bhsdi->bhsdi_status = io->scsiio.scsi_status;
2670                                 io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
2671                         }
2672                 }
2673                 KASSERT(response->ip_data_len > 0, ("sending empty Data-In"));
2674                 PRIV_EXPDATASN(io) += howmany(response->ip_data_len,
2675                     cs->cs_conn->ic_max_send_data_segment_length);
2676                 if (cb != NULL) {
2677                         response->ip_prv0 = io->scsiio.kern_data_ref;
2678                         response->ip_prv1 = io->scsiio.kern_data_arg;
2679                         io->scsiio.kern_data_ref(io->scsiio.kern_data_arg, 1);
2680                 }
2681                 cfiscsi_pdu_queue_cb(response, cb);
2682         }
2683
2684         ctl_datamove_done(io, true);
2685 }
2686
2687 static void
2688 cfiscsi_datamove_out(union ctl_io *io)
2689 {
2690         struct cfiscsi_session *cs;
2691         struct icl_pdu *request, *response;
2692         const struct iscsi_bhs_scsi_command *bhssc;
2693         struct iscsi_bhs_r2t *bhsr2t;
2694         struct cfiscsi_data_wait *cdw;
2695         struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
2696         uint32_t expected_len, datamove_len, r2t_off, r2t_len;
2697         uint32_t target_transfer_tag;
2698         bool done;
2699
2700         request = PRIV_REQUEST(io);
2701         cs = PDU_SESSION(request);
2702
2703         bhssc = (const struct iscsi_bhs_scsi_command *)request->ip_bhs;
2704         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2705             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2706             ("bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_COMMAND"));
2707
2708         /*
2709          * Complete write underflow.  Not a single byte to read.  Return.
2710          */
2711         expected_len = ntohl(bhssc->bhssc_expected_data_transfer_length);
2712         if (io->scsiio.kern_rel_offset >= expected_len) {
2713                 ctl_datamove_done(io, true);
2714                 return;
2715         }
2716
2717         datamove_len = MIN(io->scsiio.kern_data_len,
2718             expected_len - io->scsiio.kern_rel_offset);
2719
2720         target_transfer_tag =
2721             atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2722         if (target_transfer_tag == 0xffffffff) {
2723                 target_transfer_tag =
2724                     atomic_fetchadd_32(&cs->cs_target_transfer_tag, 1);
2725         }
2726         cdw = cfiscsi_data_wait_new(cs, io, bhssc->bhssc_initiator_task_tag,
2727             &target_transfer_tag);
2728         if (cdw == NULL) {
2729                 CFISCSI_SESSION_WARN(cs, "failed to "
2730                     "allocate memory; dropping connection");
2731                 ctl_set_busy(&io->scsiio);
2732                 ctl_datamove_done(io, true);
2733                 cfiscsi_session_terminate(cs);
2734                 return;
2735         }
2736 #if 0
2737         CFISCSI_SESSION_DEBUG(cs, "expecting Data-Out with initiator "
2738             "task tag 0x%x, target transfer tag 0x%x",
2739             bhssc->bhssc_initiator_task_tag, target_transfer_tag);
2740 #endif
2741
2742         cdw->cdw_ctl_io = io;
2743         cdw->cdw_target_transfer_tag = target_transfer_tag;
2744         cdw->cdw_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2745         cdw->cdw_r2t_end = datamove_len;
2746         cdw->cdw_datasn = 0;
2747
2748         /* Set initial data pointer for the CDW respecting ext_data_filled. */
2749         if (io->scsiio.kern_sg_entries > 0) {
2750                 ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
2751         } else {
2752                 ctl_sglist = &ctl_sg_entry;
2753                 ctl_sglist->addr = io->scsiio.kern_data_ptr;
2754                 ctl_sglist->len = datamove_len;
2755         }
2756         cdw->cdw_sg_index = 0;
2757         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2758         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2759         r2t_off = io->scsiio.ext_data_filled;
2760         while (r2t_off > 0) {
2761                 if (r2t_off >= cdw->cdw_sg_len) {
2762                         r2t_off -= cdw->cdw_sg_len;
2763                         cdw->cdw_sg_index++;
2764                         cdw->cdw_sg_addr = ctl_sglist[cdw->cdw_sg_index].addr;
2765                         cdw->cdw_sg_len = ctl_sglist[cdw->cdw_sg_index].len;
2766                         continue;
2767                 }
2768                 cdw->cdw_sg_addr += r2t_off;
2769                 cdw->cdw_sg_len -= r2t_off;
2770                 r2t_off = 0;
2771         }
2772
2773         if (cs->cs_immediate_data &&
2774             io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled <
2775             icl_pdu_data_segment_length(request)) {
2776                 done = cfiscsi_handle_data_segment(request, cdw);
2777                 if (done) {
2778                         cfiscsi_data_wait_free(cs, cdw);
2779                         ctl_datamove_done(io, true);
2780                         return;
2781                 }
2782         }
2783
2784         r2t_off = io->scsiio.kern_rel_offset + io->scsiio.ext_data_filled;
2785         r2t_len = MIN(datamove_len - io->scsiio.ext_data_filled,
2786             cs->cs_max_burst_length);
2787         cdw->cdw_r2t_end = io->scsiio.ext_data_filled + r2t_len;
2788
2789         CFISCSI_SESSION_LOCK(cs);
2790         if (cs->cs_terminating_tasks) {
2791                 CFISCSI_SESSION_UNLOCK(cs);
2792                 KASSERT((io->io_hdr.flags & CTL_FLAG_ABORT) != 0,
2793                     ("%s: I/O request %p on termating session %p not aborted",
2794                     __func__, io, cs));
2795                 CFISCSI_SESSION_WARN(cs, "aborting data_wait for aborted I/O");
2796                 cfiscsi_data_wait_abort(cs, cdw, 44);
2797                 return;
2798         }
2799         TAILQ_INSERT_TAIL(&cs->cs_waiting_for_data_out, cdw, cdw_next);
2800         CFISCSI_SESSION_UNLOCK(cs);
2801
2802         /*
2803          * XXX: We should limit the number of outstanding R2T PDUs
2804          *      per task to MaxOutstandingR2T.
2805          */
2806         response = cfiscsi_pdu_new_response(request, M_NOWAIT);
2807         if (response == NULL) {
2808                 CFISCSI_SESSION_WARN(cs, "failed to "
2809                     "allocate memory; dropping connection");
2810                 ctl_set_busy(&io->scsiio);
2811                 ctl_datamove_done(io, true);
2812                 cfiscsi_session_terminate(cs);
2813                 return;
2814         }
2815         io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
2816         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
2817         bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
2818         bhsr2t->bhsr2t_flags = 0x80;
2819         bhsr2t->bhsr2t_lun = bhssc->bhssc_lun;
2820         bhsr2t->bhsr2t_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2821         bhsr2t->bhsr2t_target_transfer_tag = target_transfer_tag;
2822         /*
2823          * XXX: Here we assume that cfiscsi_datamove() won't ever
2824          *      be running concurrently on several CPUs for a given
2825          *      command.
2826          */
2827         bhsr2t->bhsr2t_r2tsn = htonl(PRIV_R2TSN(io)++);
2828         /*
2829          * This is the offset within the current SCSI command;
2830          * i.e. for the first call of datamove(), it will be 0,
2831          * and for subsequent ones it will be the sum of lengths
2832          * of previous ones.
2833          *
2834          * The ext_data_filled is to account for unsolicited
2835          * (immediate) data that might have already arrived.
2836          */
2837         bhsr2t->bhsr2t_buffer_offset = htonl(r2t_off);
2838         /*
2839          * This is the total length (sum of S/G lengths) this call
2840          * to cfiscsi_datamove() is supposed to handle, limited by
2841          * MaxBurstLength.
2842          */
2843         bhsr2t->bhsr2t_desired_data_transfer_length = htonl(r2t_len);
2844         cfiscsi_pdu_queue(response);
2845 }
2846
2847 static void
2848 cfiscsi_datamove(union ctl_io *io)
2849 {
2850
2851         if ((io->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN)
2852                 cfiscsi_datamove_in(io);
2853         else {
2854                 /* We hadn't received anything during this datamove yet. */
2855                 io->scsiio.ext_data_filled = 0;
2856                 cfiscsi_datamove_out(io);
2857         }
2858 }
2859
2860 static void
2861 cfiscsi_scsi_command_done(union ctl_io *io)
2862 {
2863         struct icl_pdu *request, *response;
2864         struct iscsi_bhs_scsi_command *bhssc;
2865         struct iscsi_bhs_scsi_response *bhssr;
2866 #ifdef DIAGNOSTIC
2867         struct cfiscsi_data_wait *cdw;
2868         struct cfiscsi_session *cs;
2869 #endif
2870         uint16_t sense_length;
2871
2872         request = PRIV_REQUEST(io);
2873         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2874         KASSERT((bhssc->bhssc_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2875             ISCSI_BHS_OPCODE_SCSI_COMMAND,
2876             ("replying to wrong opcode 0x%x", bhssc->bhssc_opcode));
2877
2878         //CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x",
2879         //    bhssc->bhssc_initiator_task_tag);
2880
2881 #ifdef DIAGNOSTIC
2882         cs = PDU_SESSION(request);
2883         CFISCSI_SESSION_LOCK(cs);
2884         TAILQ_FOREACH(cdw, &cs->cs_waiting_for_data_out, cdw_next)
2885                 KASSERT(bhssc->bhssc_initiator_task_tag !=
2886                     cdw->cdw_initiator_task_tag, ("dangling cdw"));
2887         CFISCSI_SESSION_UNLOCK(cs);
2888 #endif
2889
2890         /*
2891          * Do not return status for aborted commands.
2892          * There are exceptions, but none supported by CTL yet.
2893          */
2894         if (((io->io_hdr.flags & CTL_FLAG_ABORT) &&
2895              (io->io_hdr.flags & CTL_FLAG_ABORT_STATUS) == 0) ||
2896             (io->io_hdr.flags & CTL_FLAG_STATUS_SENT)) {
2897                 ctl_free_io(io);
2898                 icl_pdu_free(request);
2899                 return;
2900         }
2901
2902         response = cfiscsi_pdu_new_response(request, M_WAITOK);
2903         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
2904         bhssr->bhssr_opcode = ISCSI_BHS_OPCODE_SCSI_RESPONSE;
2905         bhssr->bhssr_flags = 0x80;
2906         /*
2907          * XXX: We don't deal with bidirectional under/overflows;
2908          *      does anything actually support those?
2909          */
2910         if (io->scsiio.kern_total_len <
2911             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2912                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_UNDERFLOW;
2913                 bhssr->bhssr_residual_count =
2914                     htonl(ntohl(bhssc->bhssc_expected_data_transfer_length) -
2915                     io->scsiio.kern_total_len);
2916                 //CFISCSI_SESSION_DEBUG(cs, "underflow; residual count %d",
2917                 //    ntohl(bhssr->bhssr_residual_count));
2918         } else if (io->scsiio.kern_total_len >
2919             ntohl(bhssc->bhssc_expected_data_transfer_length)) {
2920                 bhssr->bhssr_flags |= BHSSR_FLAGS_RESIDUAL_OVERFLOW;
2921                 bhssr->bhssr_residual_count = htonl(io->scsiio.kern_total_len -
2922                     ntohl(bhssc->bhssc_expected_data_transfer_length));
2923                 //CFISCSI_SESSION_DEBUG(cs, "overflow; residual count %d",
2924                 //    ntohl(bhssr->bhssr_residual_count));
2925         }
2926         bhssr->bhssr_response = BHSSR_RESPONSE_COMMAND_COMPLETED;
2927         bhssr->bhssr_status = io->scsiio.scsi_status;
2928         bhssr->bhssr_initiator_task_tag = bhssc->bhssc_initiator_task_tag;
2929         bhssr->bhssr_expdatasn = htonl(PRIV_EXPDATASN(io));
2930
2931         if (io->scsiio.sense_len > 0) {
2932 #if 0
2933                 CFISCSI_SESSION_DEBUG(cs, "returning %d bytes of sense data",
2934                     io->scsiio.sense_len);
2935 #endif
2936                 sense_length = htons(io->scsiio.sense_len);
2937                 icl_pdu_append_data(response,
2938                     &sense_length, sizeof(sense_length), M_WAITOK);
2939                 icl_pdu_append_data(response,
2940                     &io->scsiio.sense_data, io->scsiio.sense_len, M_WAITOK);
2941         }
2942
2943         ctl_free_io(io);
2944         icl_pdu_free(request);
2945         cfiscsi_pdu_queue(response);
2946 }
2947
2948 static void
2949 cfiscsi_task_management_done(union ctl_io *io)
2950 {
2951         struct icl_pdu *request, *response;
2952         struct iscsi_bhs_task_management_request *bhstmr;
2953         struct iscsi_bhs_task_management_response *bhstmr2;
2954         struct cfiscsi_data_wait *cdw, *tmpcdw;
2955         struct cfiscsi_session *cs, *tcs;
2956         struct cfiscsi_softc *softc;
2957         int cold_reset = 0;
2958
2959         request = PRIV_REQUEST(io);
2960         cs = PDU_SESSION(request);
2961         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2962         KASSERT((bhstmr->bhstmr_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) ==
2963             ISCSI_BHS_OPCODE_TASK_REQUEST,
2964             ("replying to wrong opcode 0x%x", bhstmr->bhstmr_opcode));
2965
2966 #if 0
2967         CFISCSI_SESSION_DEBUG(cs, "initiator task tag 0x%x; referenced task tag 0x%x",
2968             bhstmr->bhstmr_initiator_task_tag,
2969             bhstmr->bhstmr_referenced_task_tag);
2970 #endif
2971
2972         if ((bhstmr->bhstmr_function & ~0x80) ==
2973             BHSTMR_FUNCTION_ABORT_TASK) {
2974                 /*
2975                  * Make sure we no longer wait for Data-Out for this command.
2976                  */
2977                 CFISCSI_SESSION_LOCK(cs);
2978                 TAILQ_FOREACH_SAFE(cdw,
2979                     &cs->cs_waiting_for_data_out, cdw_next, tmpcdw) {
2980                         if (bhstmr->bhstmr_referenced_task_tag !=
2981                             cdw->cdw_initiator_task_tag)
2982                                 continue;
2983
2984 #if 0
2985                         CFISCSI_SESSION_DEBUG(cs, "removing csw for initiator task "
2986                             "tag 0x%x", bhstmr->bhstmr_initiator_task_tag);
2987 #endif
2988                         TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
2989                             cdw, cdw_next);
2990                         cfiscsi_data_wait_abort(cs, cdw, 43);
2991                 }
2992                 CFISCSI_SESSION_UNLOCK(cs);
2993         }
2994         if ((bhstmr->bhstmr_function & ~0x80) ==
2995             BHSTMR_FUNCTION_TARGET_COLD_RESET &&
2996             io->io_hdr.status == CTL_SUCCESS)
2997                 cold_reset = 1;
2998
2999         response = cfiscsi_pdu_new_response(request, M_WAITOK);
3000         bhstmr2 = (struct iscsi_bhs_task_management_response *)
3001             response->ip_bhs;
3002         bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
3003         bhstmr2->bhstmr_flags = 0x80;
3004         switch (io->taskio.task_status) {
3005         case CTL_TASK_FUNCTION_COMPLETE:
3006                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
3007                 break;
3008         case CTL_TASK_FUNCTION_SUCCEEDED:
3009                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
3010                 break;
3011         case CTL_TASK_LUN_DOES_NOT_EXIST:
3012                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
3013                 break;
3014         case CTL_TASK_FUNCTION_NOT_SUPPORTED:
3015         default:
3016                 bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
3017                 break;
3018         }
3019         memcpy(bhstmr2->bhstmr_additional_reponse_information,
3020             io->taskio.task_resp, sizeof(io->taskio.task_resp));
3021         bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
3022
3023         ctl_free_io(io);
3024         icl_pdu_free(request);
3025         cfiscsi_pdu_queue(response);
3026
3027         if (cold_reset) {
3028                 softc = cs->cs_target->ct_softc;
3029                 mtx_lock(&softc->lock);
3030                 TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
3031                         if (tcs->cs_target == cs->cs_target)
3032                                 cfiscsi_session_terminate(tcs);
3033                 }
3034                 mtx_unlock(&softc->lock);
3035         }
3036 }
3037
3038 static void
3039 cfiscsi_done(union ctl_io *io)
3040 {
3041         struct icl_pdu *request;
3042         struct cfiscsi_session *cs;
3043
3044         KASSERT(((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE),
3045                 ("invalid CTL status %#x", io->io_hdr.status));
3046
3047         request = PRIV_REQUEST(io);
3048         cs = PDU_SESSION(request);
3049
3050         switch (request->ip_bhs->bhs_opcode & ~ISCSI_BHS_OPCODE_IMMEDIATE) {
3051         case ISCSI_BHS_OPCODE_SCSI_COMMAND:
3052                 cfiscsi_scsi_command_done(io);
3053                 break;
3054         case ISCSI_BHS_OPCODE_TASK_REQUEST:
3055                 cfiscsi_task_management_done(io);
3056                 break;
3057         case ISCSI_BHS_OPCODE_INTERNAL:
3058                 /*
3059                  * Implicit task termination has just completed; nothing to do.
3060                  */
3061                 icl_pdu_free(request);
3062                 cs->cs_tasks_aborted = true;
3063                 refcount_release(&cs->cs_outstanding_ctl_pdus);
3064                 wakeup(__DEVOLATILE(void *, &cs->cs_outstanding_ctl_pdus));
3065                 ctl_free_io(io);
3066                 return;
3067         default:
3068                 panic("cfiscsi_done called with wrong opcode 0x%x",
3069                     request->ip_bhs->bhs_opcode);
3070         }
3071
3072         refcount_release(&cs->cs_outstanding_ctl_pdus);
3073 }