]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/dev/iscsi/iscsi.c
cxgbe(4): Avoid unsafe hardware access in the ifmedia ioctls.
[FreeBSD/FreeBSD.git] / sys / dev / iscsi / iscsi.c
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012 The FreeBSD Foundation
5  *
6  * This software was developed by Edward Tomasz Napierala under sponsorship
7  * from the FreeBSD Foundation.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28  * SUCH DAMAGE.
29  *
30  */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 #include <sys/param.h>
36 #include <sys/condvar.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/eventhandler.h>
40 #include <sys/file.h>
41 #include <sys/kernel.h>
42 #include <sys/kthread.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mutex.h>
46 #include <sys/module.h>
47 #include <sys/socket.h>
48 #include <sys/sysctl.h>
49 #include <sys/systm.h>
50 #include <sys/sx.h>
51 #include <vm/uma.h>
52
53 #include <cam/cam.h>
54 #include <cam/cam_ccb.h>
55 #include <cam/cam_xpt.h>
56 #include <cam/cam_debug.h>
57 #include <cam/cam_sim.h>
58 #include <cam/cam_xpt_sim.h>
59 #include <cam/cam_xpt_periph.h>
60 #include <cam/cam_periph.h>
61 #include <cam/scsi/scsi_all.h>
62 #include <cam/scsi/scsi_message.h>
63
64 #include <dev/iscsi/icl.h>
65 #include <dev/iscsi/icl_wrappers.h>
66 #include <dev/iscsi/iscsi_ioctl.h>
67 #include <dev/iscsi/iscsi_proto.h>
68 #include <dev/iscsi/iscsi.h>
69
70 #ifdef ICL_KERNEL_PROXY
71 #include <sys/socketvar.h>
72 #endif
73
74 #ifdef ICL_KERNEL_PROXY
75 FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
76 #endif
77
78 /*
79  * XXX: This is global so the iscsi_unload() can access it.
80  *      Think about how to do this properly.
81  */
82 static struct iscsi_softc       *sc;
83
84 SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
85     "iSCSI initiator");
86 static int debug = 1;
87 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
88     &debug, 0, "Enable debug messages");
89 static int ping_timeout = 5;
90 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
91     0, "Timeout for ping (NOP-Out) requests, in seconds");
92 static int iscsid_timeout = 60;
93 SYSCTL_INT(_kern_iscsi, OID_AUTO, iscsid_timeout, CTLFLAG_RWTUN, &iscsid_timeout,
94     0, "Time to wait for iscsid(8) to handle reconnection, in seconds");
95 static int login_timeout = 60;
96 SYSCTL_INT(_kern_iscsi, OID_AUTO, login_timeout, CTLFLAG_RWTUN, &login_timeout,
97     0, "Time to wait for iscsid(8) to finish Login Phase, in seconds");
98 static int maxtags = 255;
99 SYSCTL_INT(_kern_iscsi, OID_AUTO, maxtags, CTLFLAG_RWTUN, &maxtags,
100     0, "Max number of IO requests queued");
101 static int fail_on_disconnection = 0;
102 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_disconnection, CTLFLAG_RWTUN,
103     &fail_on_disconnection, 0, "Destroy CAM SIM on connection failure");
104 static int fail_on_shutdown = 1;
105 SYSCTL_INT(_kern_iscsi, OID_AUTO, fail_on_shutdown, CTLFLAG_RWTUN,
106     &fail_on_shutdown, 0, "Fail disconnected sessions on shutdown");
107
108 static MALLOC_DEFINE(M_ISCSI, "iSCSI", "iSCSI initiator");
109 static uma_zone_t iscsi_outstanding_zone;
110
111 #define CONN_SESSION(X) ((struct iscsi_session *)X->ic_prv0)
112 #define PDU_SESSION(X)  (CONN_SESSION(X->ip_conn))
113
114 #define ISCSI_DEBUG(X, ...)                                             \
115         do {                                                            \
116                 if (debug > 1)                                          \
117                         printf("%s: " X "\n", __func__, ## __VA_ARGS__);\
118         } while (0)
119
120 #define ISCSI_WARN(X, ...)                                              \
121         do {                                                            \
122                 if (debug > 0) {                                        \
123                         printf("WARNING: %s: " X "\n",                  \
124                             __func__, ## __VA_ARGS__);                  \
125                 }                                                       \
126         } while (0)
127
128 #define ISCSI_SESSION_DEBUG(S, X, ...)                                  \
129         do {                                                            \
130                 if (debug > 1) {                                        \
131                         printf("%s: %s (%s): " X "\n",                  \
132                             __func__, S->is_conf.isc_target_addr,       \
133                             S->is_conf.isc_target, ## __VA_ARGS__);     \
134                 }                                                       \
135         } while (0)
136
137 #define ISCSI_SESSION_WARN(S, X, ...)                                   \
138         do {                                                            \
139                 if (debug > 0) {                                        \
140                         printf("WARNING: %s (%s): " X "\n",             \
141                             S->is_conf.isc_target_addr,                 \
142                             S->is_conf.isc_target, ## __VA_ARGS__);     \
143                 }                                                       \
144         } while (0)
145
146 #define ISCSI_SESSION_LOCK(X)           mtx_lock(&X->is_lock)
147 #define ISCSI_SESSION_UNLOCK(X)         mtx_unlock(&X->is_lock)
148 #define ISCSI_SESSION_LOCK_ASSERT(X)    mtx_assert(&X->is_lock, MA_OWNED)
149 #define ISCSI_SESSION_LOCK_ASSERT_NOT(X) mtx_assert(&X->is_lock, MA_NOTOWNED)
150
151 static int      iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg,
152                     int mode, struct thread *td);
153
154 static struct cdevsw iscsi_cdevsw = {
155      .d_version = D_VERSION,
156      .d_ioctl   = iscsi_ioctl,
157      .d_name    = "iscsi",
158 };
159
160 static void     iscsi_pdu_queue_locked(struct icl_pdu *request);
161 static void     iscsi_pdu_queue(struct icl_pdu *request);
162 static void     iscsi_pdu_update_statsn(const struct icl_pdu *response);
163 static void     iscsi_pdu_handle_nop_in(struct icl_pdu *response);
164 static void     iscsi_pdu_handle_scsi_response(struct icl_pdu *response);
165 static void     iscsi_pdu_handle_task_response(struct icl_pdu *response);
166 static void     iscsi_pdu_handle_data_in(struct icl_pdu *response);
167 static void     iscsi_pdu_handle_logout_response(struct icl_pdu *response);
168 static void     iscsi_pdu_handle_r2t(struct icl_pdu *response);
169 static void     iscsi_pdu_handle_async_message(struct icl_pdu *response);
170 static void     iscsi_pdu_handle_reject(struct icl_pdu *response);
171 static void     iscsi_session_reconnect(struct iscsi_session *is);
172 static void     iscsi_session_terminate(struct iscsi_session *is);
173 static void     iscsi_action(struct cam_sim *sim, union ccb *ccb);
174 static struct iscsi_outstanding *iscsi_outstanding_find(struct iscsi_session *is,
175                     uint32_t initiator_task_tag);
176 static struct iscsi_outstanding *iscsi_outstanding_add(struct iscsi_session *is,
177                     struct icl_pdu *request, union ccb *ccb,
178                     uint32_t *initiator_task_tagp);
179 static void     iscsi_outstanding_remove(struct iscsi_session *is,
180                     struct iscsi_outstanding *io);
181
182 static bool
183 iscsi_pdu_prepare(struct icl_pdu *request)
184 {
185         struct iscsi_session *is;
186         struct iscsi_bhs_scsi_command *bhssc;
187
188         is = PDU_SESSION(request);
189
190         ISCSI_SESSION_LOCK_ASSERT(is);
191
192         /*
193          * We're only using fields common for all the request
194          * (initiator -> target) PDUs.
195          */
196         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
197
198         /*
199          * Data-Out PDU does not contain CmdSN.
200          */
201         if (bhssc->bhssc_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_OUT) {
202                 if (ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn) &&
203                     (bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0) {
204                         /*
205                          * Current MaxCmdSN prevents us from sending any more
206                          * SCSI Command PDUs to the target; postpone the PDU.
207                          * It will get resent by either iscsi_pdu_queue(),
208                          * or by maintenance thread.
209                          */
210 #if 0
211                         ISCSI_SESSION_DEBUG(is, "postponing send, CmdSN %u, "
212                             "ExpCmdSN %u, MaxCmdSN %u, opcode 0x%x",
213                             is->is_cmdsn, is->is_expcmdsn, is->is_maxcmdsn,
214                             bhssc->bhssc_opcode);
215 #endif
216                         return (true);
217                 }
218                 bhssc->bhssc_cmdsn = htonl(is->is_cmdsn);
219                 if ((bhssc->bhssc_opcode & ISCSI_BHS_OPCODE_IMMEDIATE) == 0)
220                         is->is_cmdsn++;
221         }
222         bhssc->bhssc_expstatsn = htonl(is->is_statsn + 1);
223
224         return (false);
225 }
226
227 static void
228 iscsi_session_send_postponed(struct iscsi_session *is)
229 {
230         struct icl_pdu *request;
231         bool postpone;
232
233         ISCSI_SESSION_LOCK_ASSERT(is);
234
235         if (STAILQ_EMPTY(&is->is_postponed))
236                 return;
237         while ((request = STAILQ_FIRST(&is->is_postponed)) != NULL) {
238                 postpone = iscsi_pdu_prepare(request);
239                 if (postpone)
240                         return;
241                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
242                 icl_pdu_queue(request);
243         }
244         xpt_release_simq(is->is_sim, 1);
245 }
246
247 static void
248 iscsi_pdu_queue_locked(struct icl_pdu *request)
249 {
250         struct iscsi_session *is;
251         bool postpone;
252
253         is = PDU_SESSION(request);
254         ISCSI_SESSION_LOCK_ASSERT(is);
255         iscsi_session_send_postponed(is);
256         postpone = iscsi_pdu_prepare(request);
257         if (postpone) {
258                 if (STAILQ_EMPTY(&is->is_postponed))
259                         xpt_freeze_simq(is->is_sim, 1);
260                 STAILQ_INSERT_TAIL(&is->is_postponed, request, ip_next);
261                 return;
262         }
263         icl_pdu_queue(request);
264 }
265
266 static void
267 iscsi_pdu_queue(struct icl_pdu *request)
268 {
269         struct iscsi_session *is;
270
271         is = PDU_SESSION(request);
272         ISCSI_SESSION_LOCK(is);
273         iscsi_pdu_queue_locked(request);
274         ISCSI_SESSION_UNLOCK(is);
275 }
276
277 static void
278 iscsi_session_logout(struct iscsi_session *is)
279 {
280         struct icl_pdu *request;
281         struct iscsi_bhs_logout_request *bhslr;
282
283         request = icl_pdu_new(is->is_conn, M_NOWAIT);
284         if (request == NULL)
285                 return;
286
287         bhslr = (struct iscsi_bhs_logout_request *)request->ip_bhs;
288         bhslr->bhslr_opcode = ISCSI_BHS_OPCODE_LOGOUT_REQUEST;
289         bhslr->bhslr_reason = BHSLR_REASON_CLOSE_SESSION;
290         iscsi_pdu_queue_locked(request);
291 }
292
293 static void
294 iscsi_session_terminate_task(struct iscsi_session *is,
295     struct iscsi_outstanding *io, cam_status status)
296 {
297
298         ISCSI_SESSION_LOCK_ASSERT(is);
299
300         if (io->io_ccb != NULL) {
301                 io->io_ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
302                 io->io_ccb->ccb_h.status |= status;
303                 if ((io->io_ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
304                         io->io_ccb->ccb_h.status |= CAM_DEV_QFRZN;
305                         xpt_freeze_devq(io->io_ccb->ccb_h.path, 1);
306                         ISCSI_SESSION_DEBUG(is, "freezing devq");
307                 }
308                 xpt_done(io->io_ccb);
309         }
310         iscsi_outstanding_remove(is, io);
311 }
312
313 static void
314 iscsi_session_terminate_tasks(struct iscsi_session *is, cam_status status)
315 {
316         struct iscsi_outstanding *io, *tmp;
317
318         ISCSI_SESSION_LOCK_ASSERT(is);
319
320         TAILQ_FOREACH_SAFE(io, &is->is_outstanding, io_next, tmp) {
321                 iscsi_session_terminate_task(is, io, status);
322         }
323 }
324
325 static void
326 iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
327 {
328         struct icl_pdu *pdu;
329
330         ISCSI_SESSION_LOCK_ASSERT(is);
331
332         /*
333          * Don't queue any new PDUs.
334          */
335         if (is->is_sim != NULL && is->is_simq_frozen == false) {
336                 ISCSI_SESSION_DEBUG(is, "freezing");
337                 xpt_freeze_simq(is->is_sim, 1);
338                 is->is_simq_frozen = true;
339         }
340
341         /*
342          * Remove postponed PDUs.
343          */
344         if (!STAILQ_EMPTY(&is->is_postponed))
345                 xpt_release_simq(is->is_sim, 1);
346         while ((pdu = STAILQ_FIRST(&is->is_postponed)) != NULL) {
347                 STAILQ_REMOVE_HEAD(&is->is_postponed, ip_next);
348                 icl_pdu_free(pdu);
349         }
350
351         if (destroy_sim == false) {
352                 /*
353                  * Terminate SCSI tasks, asking CAM to requeue them.
354                  */
355                 iscsi_session_terminate_tasks(is, CAM_REQUEUE_REQ);
356                 return;
357         }
358
359         iscsi_session_terminate_tasks(is, CAM_DEV_NOT_THERE);
360
361         if (is->is_sim == NULL)
362                 return;
363
364         ISCSI_SESSION_DEBUG(is, "deregistering SIM");
365         xpt_async(AC_LOST_DEVICE, is->is_path, NULL);
366
367         if (is->is_simq_frozen) {
368                 is->is_simq_frozen = false;
369                 xpt_release_simq(is->is_sim, 1);
370         }
371
372         xpt_free_path(is->is_path);
373         is->is_path = NULL;
374         xpt_bus_deregister(cam_sim_path(is->is_sim));
375         cam_sim_free(is->is_sim, TRUE /*free_devq*/);
376         is->is_sim = NULL;
377         is->is_devq = NULL;
378 }
379
380 static void
381 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
382 {
383
384         icl_conn_close(is->is_conn);
385
386         ISCSI_SESSION_LOCK(is);
387
388         is->is_connected = false;
389         is->is_reconnecting = false;
390         is->is_login_phase = false;
391
392 #ifdef ICL_KERNEL_PROXY
393         if (is->is_login_pdu != NULL) {
394                 icl_pdu_free(is->is_login_pdu);
395                 is->is_login_pdu = NULL;
396         }
397         cv_signal(&is->is_login_cv);
398 #endif
399
400         if (fail_on_disconnection) {
401                 ISCSI_SESSION_DEBUG(is, "connection failed, destroying devices");
402                 iscsi_session_cleanup(is, true);
403         } else {
404                 iscsi_session_cleanup(is, false);
405         }
406
407         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
408             ("destroying session with active tasks"));
409         KASSERT(STAILQ_EMPTY(&is->is_postponed),
410             ("destroying session with postponed PDUs"));
411
412         if (is->is_conf.isc_enable == 0 && is->is_conf.isc_discovery == 0) {
413                 ISCSI_SESSION_UNLOCK(is);
414                 return;
415         }
416
417         /*
418          * Request immediate reconnection from iscsid(8).
419          */
420         //ISCSI_SESSION_DEBUG(is, "waking up iscsid(8)");
421         is->is_waiting_for_iscsid = true;
422         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
423         is->is_timeout = 0;
424         ISCSI_SESSION_UNLOCK(is);
425         cv_signal(&is->is_softc->sc_cv);
426 }
427
428 static void
429 iscsi_maintenance_thread_terminate(struct iscsi_session *is)
430 {
431         struct iscsi_softc *sc;
432
433         sc = is->is_softc;
434         sx_xlock(&sc->sc_lock);
435         TAILQ_REMOVE(&sc->sc_sessions, is, is_next);
436         sx_xunlock(&sc->sc_lock);
437
438         icl_conn_close(is->is_conn);
439         callout_drain(&is->is_callout);
440
441         ISCSI_SESSION_LOCK(is);
442
443         KASSERT(is->is_terminating, ("is_terminating == false"));
444
445 #ifdef ICL_KERNEL_PROXY
446         if (is->is_login_pdu != NULL) {
447                 icl_pdu_free(is->is_login_pdu);
448                 is->is_login_pdu = NULL;
449         }
450         cv_signal(&is->is_login_cv);
451 #endif
452
453         iscsi_session_cleanup(is, true);
454
455         KASSERT(TAILQ_EMPTY(&is->is_outstanding),
456             ("destroying session with active tasks"));
457         KASSERT(STAILQ_EMPTY(&is->is_postponed),
458             ("destroying session with postponed PDUs"));
459
460         ISCSI_SESSION_UNLOCK(is);
461
462         icl_conn_free(is->is_conn);
463         mtx_destroy(&is->is_lock);
464         cv_destroy(&is->is_maintenance_cv);
465 #ifdef ICL_KERNEL_PROXY
466         cv_destroy(&is->is_login_cv);
467 #endif
468
469         ISCSI_SESSION_DEBUG(is, "terminated");
470         free(is, M_ISCSI);
471
472         /*
473          * The iscsi_unload() routine might be waiting.
474          */
475         cv_signal(&sc->sc_cv);
476 }
477
478 static void
479 iscsi_maintenance_thread(void *arg)
480 {
481         struct iscsi_session *is = arg;
482
483         ISCSI_SESSION_LOCK(is);
484         for (;;) {
485                 if (is->is_reconnecting == false &&
486                     is->is_terminating == false &&
487                     (STAILQ_EMPTY(&is->is_postponed) ||
488                      ISCSI_SNGT(is->is_cmdsn, is->is_maxcmdsn)))
489                         cv_wait(&is->is_maintenance_cv, &is->is_lock);
490
491                 /* Terminate supersedes reconnect. */
492                 if (is->is_terminating) {
493                         ISCSI_SESSION_UNLOCK(is);
494                         iscsi_maintenance_thread_terminate(is);
495                         kthread_exit();
496                         return;
497                 }
498
499                 if (is->is_reconnecting) {
500                         ISCSI_SESSION_UNLOCK(is);
501                         iscsi_maintenance_thread_reconnect(is);
502                         ISCSI_SESSION_LOCK(is);
503                         continue;
504                 }
505
506                 iscsi_session_send_postponed(is);
507         }
508         ISCSI_SESSION_UNLOCK(is);
509 }
510
511 static void
512 iscsi_session_reconnect(struct iscsi_session *is)
513 {
514
515         /*
516          * XXX: We can't use locking here, because
517          *      it's being called from various contexts.
518          *      Hope it doesn't break anything.
519          */
520         if (is->is_reconnecting)
521                 return;
522
523         is->is_reconnecting = true;
524         cv_signal(&is->is_maintenance_cv);
525 }
526
527 static void
528 iscsi_session_terminate(struct iscsi_session *is)
529 {
530
531         if (is->is_terminating)
532                 return;
533
534         is->is_terminating = true;
535
536 #if 0
537         iscsi_session_logout(is);
538 #endif
539         cv_signal(&is->is_maintenance_cv);
540 }
541
542 static void
543 iscsi_callout(void *context)
544 {
545         struct icl_pdu *request;
546         struct iscsi_bhs_nop_out *bhsno;
547         struct iscsi_session *is;
548         bool reconnect_needed = false;
549
550         is = context;
551
552         ISCSI_SESSION_LOCK(is);
553         if (is->is_terminating) {
554                 ISCSI_SESSION_UNLOCK(is);
555                 return;
556         }
557
558         callout_schedule(&is->is_callout, 1 * hz);
559
560         if (is->is_conf.isc_enable == 0)
561                 goto out;
562
563         is->is_timeout++;
564
565         if (is->is_waiting_for_iscsid) {
566                 if (iscsid_timeout > 0 && is->is_timeout > iscsid_timeout) {
567                         ISCSI_SESSION_WARN(is, "timed out waiting for iscsid(8) "
568                             "for %d seconds; reconnecting",
569                             is->is_timeout);
570                         reconnect_needed = true;
571                 }
572                 goto out;
573         }
574
575         if (is->is_login_phase) {
576                 if (login_timeout > 0 && is->is_timeout > login_timeout) {
577                         ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
578                             "reconnecting", is->is_timeout);
579                         reconnect_needed = true;
580                 }
581                 goto out;
582         }
583
584         if (ping_timeout <= 0) {
585                 /*
586                  * Pings are disabled.  Don't send NOP-Out in this case.
587                  * Reset the timeout, to avoid triggering reconnection,
588                  * should the user decide to reenable them.
589                  */
590                 is->is_timeout = 0;
591                 goto out;
592         }
593
594         if (is->is_timeout >= ping_timeout) {
595                 ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
596                     "reconnecting", ping_timeout);
597                 reconnect_needed = true;
598                 goto out;
599         }
600
601         ISCSI_SESSION_UNLOCK(is);
602
603         /*
604          * If the ping was reset less than one second ago - which means
605          * that we've received some PDU during the last second - assume
606          * the traffic flows correctly and don't bother sending a NOP-Out.
607          *
608          * (It's 2 - one for one second, and one for incrementing is_timeout
609          * earlier in this routine.)
610          */
611         if (is->is_timeout < 2)
612                 return;
613
614         request = icl_pdu_new(is->is_conn, M_NOWAIT);
615         if (request == NULL) {
616                 ISCSI_SESSION_WARN(is, "failed to allocate PDU");
617                 return;
618         }
619         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
620         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
621             ISCSI_BHS_OPCODE_IMMEDIATE;
622         bhsno->bhsno_flags = 0x80;
623         bhsno->bhsno_target_transfer_tag = 0xffffffff;
624         iscsi_pdu_queue(request);
625         return;
626
627 out:
628         if (is->is_terminating) {
629                 ISCSI_SESSION_UNLOCK(is);
630                 return;
631         }
632
633         ISCSI_SESSION_UNLOCK(is);
634
635         if (reconnect_needed)
636                 iscsi_session_reconnect(is);
637 }
638
639 static void
640 iscsi_pdu_update_statsn(const struct icl_pdu *response)
641 {
642         const struct iscsi_bhs_data_in *bhsdi;
643         struct iscsi_session *is;
644         uint32_t expcmdsn, maxcmdsn, statsn;
645
646         is = PDU_SESSION(response);
647
648         ISCSI_SESSION_LOCK_ASSERT(is);
649
650         /*
651          * We're only using fields common for all the response
652          * (target -> initiator) PDUs.
653          */
654         bhsdi = (const struct iscsi_bhs_data_in *)response->ip_bhs;
655         /*
656          * Ok, I lied.  In case of Data-In, "The fields StatSN, Status,
657          * and Residual Count only have meaningful content if the S bit
658          * is set to 1", so we also need to check the bit specific for
659          * Data-In PDU.
660          */
661         if (bhsdi->bhsdi_opcode != ISCSI_BHS_OPCODE_SCSI_DATA_IN ||
662             (bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0) {
663                 statsn = ntohl(bhsdi->bhsdi_statsn);
664                 if (statsn != is->is_statsn && statsn != (is->is_statsn + 1)) {
665                         /* XXX: This is normal situation for MCS */
666                         ISCSI_SESSION_WARN(is, "PDU 0x%x StatSN %u != "
667                             "session ExpStatSN %u (or + 1); reconnecting",
668                             bhsdi->bhsdi_opcode, statsn, is->is_statsn);
669                         iscsi_session_reconnect(is);
670                 }
671                 if (ISCSI_SNGT(statsn, is->is_statsn))
672                         is->is_statsn = statsn;
673         }
674
675         expcmdsn = ntohl(bhsdi->bhsdi_expcmdsn);
676         maxcmdsn = ntohl(bhsdi->bhsdi_maxcmdsn);
677
678         if (ISCSI_SNLT(maxcmdsn + 1, expcmdsn)) {
679                 ISCSI_SESSION_DEBUG(is,
680                     "PDU MaxCmdSN %u + 1 < PDU ExpCmdSN %u; ignoring",
681                     maxcmdsn, expcmdsn);
682         } else {
683                 if (ISCSI_SNGT(maxcmdsn, is->is_maxcmdsn)) {
684                         is->is_maxcmdsn = maxcmdsn;
685
686                         /*
687                          * Command window increased; kick the maintanance thread
688                          * to send out postponed commands.
689                          */
690                         if (!STAILQ_EMPTY(&is->is_postponed))
691                                 cv_signal(&is->is_maintenance_cv);
692                 } else if (ISCSI_SNLT(maxcmdsn, is->is_maxcmdsn)) {
693                         /* XXX: This is normal situation for MCS */
694                         ISCSI_SESSION_DEBUG(is,
695                             "PDU MaxCmdSN %u < session MaxCmdSN %u; ignoring",
696                             maxcmdsn, is->is_maxcmdsn);
697                 }
698
699                 if (ISCSI_SNGT(expcmdsn, is->is_expcmdsn)) {
700                         is->is_expcmdsn = expcmdsn;
701                 } else if (ISCSI_SNLT(expcmdsn, is->is_expcmdsn)) {
702                         /* XXX: This is normal situation for MCS */
703                         ISCSI_SESSION_DEBUG(is,
704                             "PDU ExpCmdSN %u < session ExpCmdSN %u; ignoring",
705                             expcmdsn, is->is_expcmdsn);
706                 }
707         }
708
709         /*
710          * Every incoming PDU - not just NOP-In - resets the ping timer.
711          * The purpose of the timeout is to reset the connection when it stalls;
712          * we don't want this to happen when NOP-In or NOP-Out ends up delayed
713          * in some queue.
714          */
715         is->is_timeout = 0;
716 }
717
718 static void
719 iscsi_receive_callback(struct icl_pdu *response)
720 {
721         struct iscsi_session *is;
722
723         is = PDU_SESSION(response);
724
725         ISCSI_SESSION_LOCK(is);
726
727         iscsi_pdu_update_statsn(response);
728
729 #ifdef ICL_KERNEL_PROXY
730         if (is->is_login_phase) {
731                 if (is->is_login_pdu == NULL)
732                         is->is_login_pdu = response;
733                 else
734                         icl_pdu_free(response);
735                 ISCSI_SESSION_UNLOCK(is);
736                 cv_signal(&is->is_login_cv);
737                 return;
738         }
739 #endif
740
741         /*
742          * The handling routine is responsible for freeing the PDU
743          * when it's no longer needed.
744          */
745         switch (response->ip_bhs->bhs_opcode) {
746         case ISCSI_BHS_OPCODE_NOP_IN:
747                 iscsi_pdu_handle_nop_in(response);
748                 ISCSI_SESSION_UNLOCK(is);
749                 break;
750         case ISCSI_BHS_OPCODE_SCSI_RESPONSE:
751                 iscsi_pdu_handle_scsi_response(response);
752                 /* Session lock dropped inside. */
753                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
754                 break;
755         case ISCSI_BHS_OPCODE_TASK_RESPONSE:
756                 iscsi_pdu_handle_task_response(response);
757                 ISCSI_SESSION_UNLOCK(is);
758                 break;
759         case ISCSI_BHS_OPCODE_SCSI_DATA_IN:
760                 iscsi_pdu_handle_data_in(response);
761                 /* Session lock dropped inside. */
762                 ISCSI_SESSION_LOCK_ASSERT_NOT(is);
763                 break;
764         case ISCSI_BHS_OPCODE_LOGOUT_RESPONSE:
765                 iscsi_pdu_handle_logout_response(response);
766                 ISCSI_SESSION_UNLOCK(is);
767                 break;
768         case ISCSI_BHS_OPCODE_R2T:
769                 iscsi_pdu_handle_r2t(response);
770                 ISCSI_SESSION_UNLOCK(is);
771                 break;
772         case ISCSI_BHS_OPCODE_ASYNC_MESSAGE:
773                 iscsi_pdu_handle_async_message(response);
774                 ISCSI_SESSION_UNLOCK(is);
775                 break;
776         case ISCSI_BHS_OPCODE_REJECT:
777                 iscsi_pdu_handle_reject(response);
778                 ISCSI_SESSION_UNLOCK(is);
779                 break;
780         default:
781                 ISCSI_SESSION_WARN(is, "received PDU with unsupported "
782                     "opcode 0x%x; reconnecting",
783                     response->ip_bhs->bhs_opcode);
784                 iscsi_session_reconnect(is);
785                 ISCSI_SESSION_UNLOCK(is);
786                 icl_pdu_free(response);
787         }
788 }
789
790 static void
791 iscsi_error_callback(struct icl_conn *ic)
792 {
793         struct iscsi_session *is;
794
795         is = CONN_SESSION(ic);
796
797         ISCSI_SESSION_WARN(is, "connection error; reconnecting");
798         iscsi_session_reconnect(is);
799 }
800
801 static void
802 iscsi_pdu_handle_nop_in(struct icl_pdu *response)
803 {
804         struct iscsi_session *is;
805         struct iscsi_bhs_nop_out *bhsno;
806         struct iscsi_bhs_nop_in *bhsni;
807         struct icl_pdu *request;
808         void *data = NULL;
809         size_t datasize;
810         int error;
811
812         is = PDU_SESSION(response);
813         bhsni = (struct iscsi_bhs_nop_in *)response->ip_bhs;
814
815         if (bhsni->bhsni_target_transfer_tag == 0xffffffff) {
816                 /*
817                  * Nothing to do; iscsi_pdu_update_statsn() already
818                  * zeroed the timeout.
819                  */
820                 icl_pdu_free(response);
821                 return;
822         }
823
824         datasize = icl_pdu_data_segment_length(response);
825         if (datasize > 0) {
826                 data = malloc(datasize, M_ISCSI, M_NOWAIT | M_ZERO);
827                 if (data == NULL) {
828                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
829                             "reconnecting");
830                         icl_pdu_free(response);
831                         iscsi_session_reconnect(is);
832                         return;
833                 }
834                 icl_pdu_get_data(response, 0, data, datasize);
835         }
836
837         request = icl_pdu_new(response->ip_conn, M_NOWAIT);
838         if (request == NULL) {
839                 ISCSI_SESSION_WARN(is, "failed to allocate memory; "
840                     "reconnecting");
841                 free(data, M_ISCSI);
842                 icl_pdu_free(response);
843                 iscsi_session_reconnect(is);
844                 return;
845         }
846         bhsno = (struct iscsi_bhs_nop_out *)request->ip_bhs;
847         bhsno->bhsno_opcode = ISCSI_BHS_OPCODE_NOP_OUT |
848             ISCSI_BHS_OPCODE_IMMEDIATE;
849         bhsno->bhsno_flags = 0x80;
850         bhsno->bhsno_initiator_task_tag = 0xffffffff;
851         bhsno->bhsno_target_transfer_tag = bhsni->bhsni_target_transfer_tag;
852         if (datasize > 0) {
853                 error = icl_pdu_append_data(request, data, datasize, M_NOWAIT);
854                 if (error != 0) {
855                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
856                             "reconnecting");
857                         free(data, M_ISCSI);
858                         icl_pdu_free(request);
859                         icl_pdu_free(response);
860                         iscsi_session_reconnect(is);
861                         return;
862                 }
863                 free(data, M_ISCSI);
864         }
865
866         icl_pdu_free(response);
867         iscsi_pdu_queue_locked(request);
868 }
869
870 static void
871 iscsi_pdu_handle_scsi_response(struct icl_pdu *response)
872 {
873         struct iscsi_bhs_scsi_response *bhssr;
874         struct iscsi_outstanding *io;
875         struct iscsi_session *is;
876         union ccb *ccb;
877         struct ccb_scsiio *csio;
878         size_t data_segment_len, received;
879         uint16_t sense_len;
880         uint32_t resid;
881
882         is = PDU_SESSION(response);
883
884         bhssr = (struct iscsi_bhs_scsi_response *)response->ip_bhs;
885         io = iscsi_outstanding_find(is, bhssr->bhssr_initiator_task_tag);
886         if (io == NULL || io->io_ccb == NULL) {
887                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhssr->bhssr_initiator_task_tag);
888                 icl_pdu_free(response);
889                 iscsi_session_reconnect(is);
890                 ISCSI_SESSION_UNLOCK(is);
891                 return;
892         }
893
894         ccb = io->io_ccb;
895         if (bhssr->bhssr_response == BHSSR_RESPONSE_COMMAND_COMPLETED) {
896                 if (ntohl(bhssr->bhssr_expdatasn) != io->io_datasn) {
897                         ISCSI_SESSION_WARN(is,
898                             "ExpDataSN mismatch in SCSI Response (%u vs %u)",
899                             ntohl(bhssr->bhssr_expdatasn), io->io_datasn);
900
901                         /*
902                          * XXX: Permit an ExpDataSN of zero for errors.
903                          *
904                          * This doesn't conform to RFC 7143, but some
905                          * targets seem to do this.
906                          */
907                         if (bhssr->bhssr_status != 0 &&
908                             bhssr->bhssr_expdatasn == htonl(0))
909                                 goto skip_expdatasn;
910
911                         icl_pdu_free(response);
912                         iscsi_session_reconnect(is);
913                         ISCSI_SESSION_UNLOCK(is);
914                         return;
915                 }
916         } else {
917                 if (bhssr->bhssr_expdatasn != htonl(0)) {
918                         ISCSI_SESSION_WARN(is,
919                             "ExpDataSN mismatch in SCSI Response (%u vs 0)",
920                             ntohl(bhssr->bhssr_expdatasn));
921                         icl_pdu_free(response);
922                         iscsi_session_reconnect(is);
923                         ISCSI_SESSION_UNLOCK(is);
924                         return;
925                 }
926         }
927 skip_expdatasn:
928
929         /*
930          * With iSER, after getting good response we can be sure
931          * that all the data has been successfully transferred.
932          */
933         if (is->is_conn->ic_iser) {
934                 resid = ntohl(bhssr->bhssr_residual_count);
935                 if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW) {
936                         io->io_received = ccb->csio.dxfer_len - resid;
937                 } else if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_OVERFLOW) {
938                         ISCSI_SESSION_WARN(is, "overflow: target indicates %d", resid);
939                 } else {
940                         io->io_received = ccb->csio.dxfer_len;
941                 }
942         }
943
944         received = io->io_received;
945         iscsi_outstanding_remove(is, io);
946         ISCSI_SESSION_UNLOCK(is);
947
948         if (bhssr->bhssr_response != BHSSR_RESPONSE_COMMAND_COMPLETED) {
949                 ISCSI_SESSION_WARN(is, "service response 0x%x", bhssr->bhssr_response);
950                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
951                         xpt_freeze_devq(ccb->ccb_h.path, 1);
952                         ISCSI_SESSION_DEBUG(is, "freezing devq");
953                 }
954                 ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
955         } else if (bhssr->bhssr_status == 0) {
956                 ccb->ccb_h.status = CAM_REQ_CMP;
957         } else {
958                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
959                         xpt_freeze_devq(ccb->ccb_h.path, 1);
960                         ISCSI_SESSION_DEBUG(is, "freezing devq");
961                 }
962                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
963                 ccb->csio.scsi_status = bhssr->bhssr_status;
964         }
965
966         csio = &ccb->csio;
967         data_segment_len = icl_pdu_data_segment_length(response);
968         if (data_segment_len > 0) {
969                 if (data_segment_len < sizeof(sense_len)) {
970                         ISCSI_SESSION_WARN(is, "truncated data segment (%zd bytes)",
971                             data_segment_len);
972                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
973                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
974                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
975                         }
976                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
977                         goto out;
978                 }
979                 icl_pdu_get_data(response, 0, &sense_len, sizeof(sense_len));
980                 sense_len = ntohs(sense_len);
981 #if 0
982                 ISCSI_SESSION_DEBUG(is, "sense_len %d, data len %zd",
983                     sense_len, data_segment_len);
984 #endif
985                 if (sizeof(sense_len) + sense_len > data_segment_len) {
986                         ISCSI_SESSION_WARN(is, "truncated data segment "
987                             "(%zd bytes, should be %zd)",
988                             data_segment_len, sizeof(sense_len) + sense_len);
989                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
990                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
991                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
992                         }
993                         ccb->ccb_h.status = CAM_REQ_CMP_ERR | CAM_DEV_QFRZN;
994                         goto out;
995                 } else if (sizeof(sense_len) + sense_len < data_segment_len)
996                         ISCSI_SESSION_WARN(is, "oversize data segment "
997                             "(%zd bytes, should be %zd)",
998                             data_segment_len, sizeof(sense_len) + sense_len);
999                 if (sense_len > csio->sense_len) {
1000                         ISCSI_SESSION_DEBUG(is, "truncating sense from %d to %d",
1001                             sense_len, csio->sense_len);
1002                         sense_len = csio->sense_len;
1003                 }
1004                 icl_pdu_get_data(response, sizeof(sense_len), &csio->sense_data, sense_len);
1005                 csio->sense_resid = csio->sense_len - sense_len;
1006                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
1007         }
1008
1009 out:
1010         if (bhssr->bhssr_flags & BHSSR_FLAGS_RESIDUAL_UNDERFLOW)
1011                 csio->resid = ntohl(bhssr->bhssr_residual_count);
1012
1013         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1014                 KASSERT(received <= csio->dxfer_len,
1015                     ("received > csio->dxfer_len"));
1016                 if (received < csio->dxfer_len) {
1017                         if (csio->resid != csio->dxfer_len - received) {
1018                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1019                                     "target indicates %d, we calculated %zd",
1020                                     csio->resid, csio->dxfer_len - received);
1021                         }
1022                         csio->resid = csio->dxfer_len - received;
1023                 }
1024         }
1025
1026         xpt_done(ccb);
1027         icl_pdu_free(response);
1028 }
1029
1030 static void
1031 iscsi_pdu_handle_task_response(struct icl_pdu *response)
1032 {
1033         struct iscsi_bhs_task_management_response *bhstmr;
1034         struct iscsi_outstanding *io, *aio;
1035         struct iscsi_session *is;
1036
1037         is = PDU_SESSION(response);
1038
1039         bhstmr = (struct iscsi_bhs_task_management_response *)response->ip_bhs;
1040         io = iscsi_outstanding_find(is, bhstmr->bhstmr_initiator_task_tag);
1041         if (io == NULL || io->io_ccb != NULL) {
1042                 ISCSI_SESSION_WARN(is, "bad itt 0x%x",
1043                     bhstmr->bhstmr_initiator_task_tag);
1044                 icl_pdu_free(response);
1045                 iscsi_session_reconnect(is);
1046                 return;
1047         }
1048
1049         if (bhstmr->bhstmr_response != BHSTMR_RESPONSE_FUNCTION_COMPLETE) {
1050                 ISCSI_SESSION_WARN(is, "task response 0x%x",
1051                     bhstmr->bhstmr_response);
1052         } else {
1053                 aio = iscsi_outstanding_find(is, io->io_referenced_task_tag);
1054                 if (aio != NULL && aio->io_ccb != NULL)
1055                         iscsi_session_terminate_task(is, aio, CAM_REQ_ABORTED);
1056         }
1057
1058         iscsi_outstanding_remove(is, io);
1059         icl_pdu_free(response);
1060 }
1061
1062 static void
1063 iscsi_pdu_handle_data_in(struct icl_pdu *response)
1064 {
1065         struct iscsi_bhs_data_in *bhsdi;
1066         struct iscsi_outstanding *io;
1067         struct iscsi_session *is;
1068         union ccb *ccb;
1069         struct ccb_scsiio *csio;
1070         size_t data_segment_len, received, oreceived;
1071
1072         is = PDU_SESSION(response);
1073         bhsdi = (struct iscsi_bhs_data_in *)response->ip_bhs;
1074         io = iscsi_outstanding_find(is, bhsdi->bhsdi_initiator_task_tag);
1075         if (io == NULL || io->io_ccb == NULL) {
1076                 ISCSI_SESSION_WARN(is, "bad itt 0x%x", bhsdi->bhsdi_initiator_task_tag);
1077                 icl_pdu_free(response);
1078                 iscsi_session_reconnect(is);
1079                 ISCSI_SESSION_UNLOCK(is);
1080                 return;
1081         }
1082
1083         if (io->io_datasn != ntohl(bhsdi->bhsdi_datasn)) {
1084                 ISCSI_SESSION_WARN(is, "received Data-In PDU with "
1085                     "DataSN %u, while expected %u; dropping connection",
1086                     ntohl(bhsdi->bhsdi_datasn), io->io_datasn);
1087                 icl_pdu_free(response);
1088                 iscsi_session_reconnect(is);
1089                 ISCSI_SESSION_UNLOCK(is);
1090                 return;
1091         }
1092         io->io_datasn += response->ip_additional_pdus + 1;
1093
1094         data_segment_len = icl_pdu_data_segment_length(response);
1095         if (data_segment_len == 0) {
1096                 /*
1097                  * "The sending of 0 length data segments should be avoided,
1098                  * but initiators and targets MUST be able to properly receive
1099                  * 0 length data segments."
1100                  */
1101                 ISCSI_SESSION_UNLOCK(is);
1102                 icl_pdu_free(response);
1103                 return;
1104         }
1105
1106         /*
1107          * We need to track this for security reasons - without it, malicious target
1108          * could respond to SCSI READ without sending Data-In PDUs, which would result
1109          * in read operation on the initiator side returning random kernel data.
1110          */
1111         if (ntohl(bhsdi->bhsdi_buffer_offset) != io->io_received) {
1112                 ISCSI_SESSION_WARN(is, "data out of order; expected offset %zd, got %zd",
1113                     io->io_received, (size_t)ntohl(bhsdi->bhsdi_buffer_offset));
1114                 icl_pdu_free(response);
1115                 iscsi_session_reconnect(is);
1116                 ISCSI_SESSION_UNLOCK(is);
1117                 return;
1118         }
1119
1120         ccb = io->io_ccb;
1121         csio = &ccb->csio;
1122
1123         if (io->io_received + data_segment_len > csio->dxfer_len) {
1124                 ISCSI_SESSION_WARN(is, "oversize data segment (%zd bytes "
1125                     "at offset %zd, buffer is %d)",
1126                     data_segment_len, io->io_received, csio->dxfer_len);
1127                 icl_pdu_free(response);
1128                 iscsi_session_reconnect(is);
1129                 ISCSI_SESSION_UNLOCK(is);
1130                 return;
1131         }
1132
1133         oreceived = io->io_received;
1134         io->io_received += data_segment_len;
1135         received = io->io_received;
1136         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) != 0)
1137                 iscsi_outstanding_remove(is, io);
1138         ISCSI_SESSION_UNLOCK(is);
1139
1140         icl_pdu_get_data(response, 0, csio->data_ptr + oreceived, data_segment_len);
1141
1142         /*
1143          * XXX: Check F.
1144          */
1145         if ((bhsdi->bhsdi_flags & BHSDI_FLAGS_S) == 0) {
1146                 /*
1147                  * Nothing more to do.
1148                  */
1149                 icl_pdu_free(response);
1150                 return;
1151         }
1152
1153         //ISCSI_SESSION_DEBUG(is, "got S flag; status 0x%x", bhsdi->bhsdi_status);
1154         if (bhsdi->bhsdi_status == 0) {
1155                 ccb->ccb_h.status = CAM_REQ_CMP;
1156         } else {
1157                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
1158                         xpt_freeze_devq(ccb->ccb_h.path, 1);
1159                         ISCSI_SESSION_DEBUG(is, "freezing devq");
1160                 }
1161                 ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_DEV_QFRZN;
1162                 csio->scsi_status = bhsdi->bhsdi_status;
1163         }
1164
1165         if ((csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1166                 KASSERT(received <= csio->dxfer_len,
1167                     ("received > csio->dxfer_len"));
1168                 if (received < csio->dxfer_len) {
1169                         csio->resid = ntohl(bhsdi->bhsdi_residual_count);
1170                         if (csio->resid != csio->dxfer_len - received) {
1171                                 ISCSI_SESSION_WARN(is, "underflow mismatch: "
1172                                     "target indicates %d, we calculated %zd",
1173                                     csio->resid, csio->dxfer_len - received);
1174                         }
1175                         csio->resid = csio->dxfer_len - received;
1176                 }
1177         }
1178
1179         xpt_done(ccb);
1180         icl_pdu_free(response);
1181 }
1182
1183 static void
1184 iscsi_pdu_handle_logout_response(struct icl_pdu *response)
1185 {
1186
1187         ISCSI_SESSION_DEBUG(PDU_SESSION(response), "logout response");
1188         icl_pdu_free(response);
1189 }
1190
1191 static void
1192 iscsi_pdu_handle_r2t(struct icl_pdu *response)
1193 {
1194         struct icl_pdu *request;
1195         struct iscsi_session *is;
1196         struct iscsi_bhs_r2t *bhsr2t;
1197         struct iscsi_bhs_data_out *bhsdo;
1198         struct iscsi_outstanding *io;
1199         struct ccb_scsiio *csio;
1200         size_t off, len, max_send_data_segment_length, total_len;
1201         int error;
1202         uint32_t datasn = 0;
1203
1204         is = PDU_SESSION(response);
1205
1206         bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
1207         io = iscsi_outstanding_find(is, bhsr2t->bhsr2t_initiator_task_tag);
1208         if (io == NULL || io->io_ccb == NULL) {
1209                 ISCSI_SESSION_WARN(is, "bad itt 0x%x; reconnecting",
1210                     bhsr2t->bhsr2t_initiator_task_tag);
1211                 icl_pdu_free(response);
1212                 iscsi_session_reconnect(is);
1213                 return;
1214         }
1215
1216         csio = &io->io_ccb->csio;
1217
1218         if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_OUT) {
1219                 ISCSI_SESSION_WARN(is, "received R2T for read command; reconnecting");
1220                 icl_pdu_free(response);
1221                 iscsi_session_reconnect(is);
1222                 return;
1223         }
1224
1225         /*
1226          * XXX: Verify R2TSN.
1227          */
1228
1229         off = ntohl(bhsr2t->bhsr2t_buffer_offset);
1230         if (off > csio->dxfer_len) {
1231                 ISCSI_SESSION_WARN(is, "target requested invalid offset "
1232                     "%zd, buffer is is %d; reconnecting", off, csio->dxfer_len);
1233                 icl_pdu_free(response);
1234                 iscsi_session_reconnect(is);
1235                 return;
1236         }
1237
1238         total_len = ntohl(bhsr2t->bhsr2t_desired_data_transfer_length);
1239         if (total_len == 0 || total_len > csio->dxfer_len) {
1240                 ISCSI_SESSION_WARN(is, "target requested invalid length "
1241                     "%zd, buffer is %d; reconnecting", total_len, csio->dxfer_len);
1242                 icl_pdu_free(response);
1243                 iscsi_session_reconnect(is);
1244                 return;
1245         }
1246
1247         //ISCSI_SESSION_DEBUG(is, "r2t; off %zd, len %zd", off, total_len);
1248
1249         if (is->is_conn->ic_hw_isomax != 0)
1250                 max_send_data_segment_length = is->is_conn->ic_hw_isomax;
1251         else
1252                 max_send_data_segment_length =
1253                     is->is_conn->ic_max_send_data_segment_length;
1254         for (;;) {
1255                 len = total_len;
1256
1257                 if (len > max_send_data_segment_length)
1258                         len = max_send_data_segment_length;
1259
1260                 if (off + len > csio->dxfer_len) {
1261                         ISCSI_SESSION_WARN(is, "target requested invalid "
1262                             "length/offset %zd, buffer is %d; reconnecting",
1263                             off + len, csio->dxfer_len);
1264                         icl_pdu_free(response);
1265                         iscsi_session_reconnect(is);
1266                         return;
1267                 }
1268
1269                 request = icl_pdu_new(response->ip_conn, M_NOWAIT);
1270                 if (request == NULL) {
1271                         icl_pdu_free(response);
1272                         iscsi_session_reconnect(is);
1273                         return;
1274                 }
1275
1276                 bhsdo = (struct iscsi_bhs_data_out *)request->ip_bhs;
1277                 bhsdo->bhsdo_opcode = ISCSI_BHS_OPCODE_SCSI_DATA_OUT;
1278                 bhsdo->bhsdo_lun = bhsr2t->bhsr2t_lun;
1279                 bhsdo->bhsdo_initiator_task_tag =
1280                     bhsr2t->bhsr2t_initiator_task_tag;
1281                 bhsdo->bhsdo_target_transfer_tag =
1282                     bhsr2t->bhsr2t_target_transfer_tag;
1283                 bhsdo->bhsdo_datasn = htonl(datasn);
1284                 bhsdo->bhsdo_buffer_offset = htonl(off);
1285                 error = icl_pdu_append_data(request, csio->data_ptr + off, len,
1286                     M_NOWAIT);
1287                 if (error != 0) {
1288                         ISCSI_SESSION_WARN(is, "failed to allocate memory; "
1289                             "reconnecting");
1290                         icl_pdu_free(request);
1291                         icl_pdu_free(response);
1292                         iscsi_session_reconnect(is);
1293                         return;
1294                 }
1295
1296                 datasn += howmany(len,
1297                     is->is_conn->ic_max_send_data_segment_length);
1298                 off += len;
1299                 total_len -= len;
1300
1301                 if (total_len == 0) {
1302                         bhsdo->bhsdo_flags |= BHSDO_FLAGS_F;
1303                         //ISCSI_SESSION_DEBUG(is, "setting F, off %zd", off);
1304                 } else {
1305                         //ISCSI_SESSION_DEBUG(is, "not finished, off %zd", off);
1306                 }
1307
1308                 iscsi_pdu_queue_locked(request);
1309
1310                 if (total_len == 0)
1311                         break;
1312         }
1313
1314         icl_pdu_free(response);
1315 }
1316
1317 static void
1318 iscsi_pdu_handle_async_message(struct icl_pdu *response)
1319 {
1320         struct iscsi_bhs_asynchronous_message *bhsam;
1321         struct iscsi_session *is;
1322
1323         is = PDU_SESSION(response);
1324         bhsam = (struct iscsi_bhs_asynchronous_message *)response->ip_bhs;
1325         switch (bhsam->bhsam_async_event) {
1326         case BHSAM_EVENT_TARGET_REQUESTS_LOGOUT:
1327                 ISCSI_SESSION_WARN(is, "target requests logout; removing session");
1328                 iscsi_session_logout(is);
1329                 iscsi_session_terminate(is);
1330                 break;
1331         case BHSAM_EVENT_TARGET_TERMINATES_CONNECTION:
1332                 ISCSI_SESSION_WARN(is, "target indicates it will drop the connection");
1333                 break;
1334         case BHSAM_EVENT_TARGET_TERMINATES_SESSION:
1335                 ISCSI_SESSION_WARN(is, "target indicates it will drop the session");
1336                 break;
1337         default:
1338                 /*
1339                  * XXX: Technically, we're obligated to also handle
1340                  *      parameter renegotiation.
1341                  */
1342                 ISCSI_SESSION_WARN(is, "ignoring AsyncEvent %d", bhsam->bhsam_async_event);
1343                 break;
1344         }
1345
1346         icl_pdu_free(response);
1347 }
1348
1349 static void
1350 iscsi_pdu_handle_reject(struct icl_pdu *response)
1351 {
1352         struct iscsi_bhs_reject *bhsr;
1353         struct iscsi_session *is;
1354
1355         is = PDU_SESSION(response);
1356         bhsr = (struct iscsi_bhs_reject *)response->ip_bhs;
1357         ISCSI_SESSION_WARN(is, "received Reject PDU, reason 0x%x; protocol error?",
1358             bhsr->bhsr_reason);
1359
1360         icl_pdu_free(response);
1361 }
1362
1363 static int
1364 iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
1365     struct iscsi_daemon_request *request)
1366 {
1367         struct iscsi_session *is;
1368         struct icl_drv_limits idl;
1369         int error;
1370
1371         sx_slock(&sc->sc_lock);
1372         for (;;) {
1373                 TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1374                         ISCSI_SESSION_LOCK(is);
1375                         if (is->is_conf.isc_enable == 0 &&
1376                             is->is_conf.isc_discovery == 0) {
1377                                 ISCSI_SESSION_UNLOCK(is);
1378                                 continue;
1379                         }
1380                         if (is->is_waiting_for_iscsid)
1381                                 break;
1382                         ISCSI_SESSION_UNLOCK(is);
1383                 }
1384
1385                 if (is == NULL) {
1386                         if (sc->sc_unloading) {
1387                                 sx_sunlock(&sc->sc_lock);
1388                                 return (ENXIO);
1389                         }
1390
1391                         /*
1392                          * No session requires attention from iscsid(8); wait.
1393                          */
1394                         error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
1395                         if (error != 0) {
1396                                 sx_sunlock(&sc->sc_lock);
1397                                 return (error);
1398                         }
1399                         continue;
1400                 }
1401
1402                 is->is_waiting_for_iscsid = false;
1403                 is->is_login_phase = true;
1404                 is->is_reason[0] = '\0';
1405                 ISCSI_SESSION_UNLOCK(is);
1406
1407                 request->idr_session_id = is->is_id;
1408                 memcpy(&request->idr_isid, &is->is_isid,
1409                     sizeof(request->idr_isid));
1410                 request->idr_tsih = 0;  /* New or reinstated session. */
1411                 memcpy(&request->idr_conf, &is->is_conf,
1412                     sizeof(request->idr_conf));
1413
1414                 error = icl_limits(is->is_conf.isc_offload,
1415                     is->is_conf.isc_iser, &idl);
1416                 if (error != 0) {
1417                         ISCSI_SESSION_WARN(is, "icl_limits for offload \"%s\" "
1418                             "failed with error %d", is->is_conf.isc_offload,
1419                             error);
1420                         sx_sunlock(&sc->sc_lock);
1421                         return (error);
1422                 }
1423                 request->idr_limits.isl_max_recv_data_segment_length =
1424                     idl.idl_max_recv_data_segment_length;
1425                 request->idr_limits.isl_max_send_data_segment_length =
1426                     idl.idl_max_send_data_segment_length;
1427                 request->idr_limits.isl_max_burst_length =
1428                     idl.idl_max_burst_length;
1429                 request->idr_limits.isl_first_burst_length =
1430                     idl.idl_first_burst_length;
1431
1432                 sx_sunlock(&sc->sc_lock);
1433                 return (0);
1434         }
1435 }
1436
1437 static int
1438 iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
1439     struct iscsi_daemon_handoff *handoff)
1440 {
1441         struct iscsi_session *is;
1442         struct icl_conn *ic;
1443         int error;
1444
1445         sx_slock(&sc->sc_lock);
1446
1447         /*
1448          * Find the session to hand off socket to.
1449          */
1450         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1451                 if (is->is_id == handoff->idh_session_id)
1452                         break;
1453         }
1454         if (is == NULL) {
1455                 sx_sunlock(&sc->sc_lock);
1456                 return (ESRCH);
1457         }
1458         ISCSI_SESSION_LOCK(is);
1459         ic = is->is_conn;
1460         if (is->is_conf.isc_discovery || is->is_terminating) {
1461                 ISCSI_SESSION_UNLOCK(is);
1462                 sx_sunlock(&sc->sc_lock);
1463                 return (EINVAL);
1464         }
1465         if (is->is_connected) {
1466                 /*
1467                  * This might have happened because another iscsid(8)
1468                  * instance handed off the connection in the meantime.
1469                  * Just return.
1470                  */
1471                 ISCSI_SESSION_WARN(is, "handoff on already connected "
1472                     "session");
1473                 ISCSI_SESSION_UNLOCK(is);
1474                 sx_sunlock(&sc->sc_lock);
1475                 return (EBUSY);
1476         }
1477
1478         strlcpy(is->is_target_alias, handoff->idh_target_alias,
1479             sizeof(is->is_target_alias));
1480         is->is_tsih = handoff->idh_tsih;
1481         is->is_statsn = handoff->idh_statsn;
1482         is->is_protocol_level = handoff->idh_protocol_level;
1483         is->is_initial_r2t = handoff->idh_initial_r2t;
1484         is->is_immediate_data = handoff->idh_immediate_data;
1485
1486         ic->ic_max_recv_data_segment_length =
1487             handoff->idh_max_recv_data_segment_length;
1488         ic->ic_max_send_data_segment_length =
1489             handoff->idh_max_send_data_segment_length;
1490         is->is_max_burst_length = handoff->idh_max_burst_length;
1491         is->is_first_burst_length = handoff->idh_first_burst_length;
1492
1493         if (handoff->idh_header_digest == ISCSI_DIGEST_CRC32C)
1494                 ic->ic_header_crc32c = true;
1495         else
1496                 ic->ic_header_crc32c = false;
1497         if (handoff->idh_data_digest == ISCSI_DIGEST_CRC32C)
1498                 ic->ic_data_crc32c = true;
1499         else
1500                 ic->ic_data_crc32c = false;
1501         ic->ic_maxtags = maxtags;
1502
1503         is->is_cmdsn = 0;
1504         is->is_expcmdsn = 0;
1505         is->is_maxcmdsn = 0;
1506         is->is_waiting_for_iscsid = false;
1507         is->is_login_phase = false;
1508         is->is_timeout = 0;
1509         is->is_connected = true;
1510         is->is_reason[0] = '\0';
1511
1512         ISCSI_SESSION_UNLOCK(is);
1513
1514         /*
1515          * If we're going through the proxy, the idh_socket will be 0,
1516          * and the ICL module can simply ignore this call.  It can also
1517          * use it to determine it's no longer in the Login phase.
1518          */
1519         error = icl_conn_handoff(ic, handoff->idh_socket);
1520         if (error != 0) {
1521                 sx_sunlock(&sc->sc_lock);
1522                 iscsi_session_terminate(is);
1523                 return (error);
1524         }
1525
1526         sx_sunlock(&sc->sc_lock);
1527
1528         if (is->is_sim != NULL) {
1529                 /*
1530                  * When reconnecting, there already is SIM allocated for the session.
1531                  */
1532                 KASSERT(is->is_simq_frozen, ("reconnect without frozen simq"));
1533                 ISCSI_SESSION_LOCK(is);
1534                 ISCSI_SESSION_DEBUG(is, "releasing");
1535                 is->is_simq_frozen = false;
1536                 xpt_release_simq(is->is_sim, 1);
1537                 ISCSI_SESSION_UNLOCK(is);
1538
1539         } else {
1540                 ISCSI_SESSION_LOCK(is);
1541                 is->is_devq = cam_simq_alloc(ic->ic_maxtags);
1542                 if (is->is_devq == NULL) {
1543                         ISCSI_SESSION_UNLOCK(is);
1544                         ISCSI_SESSION_WARN(is, "failed to allocate simq");
1545                         iscsi_session_terminate(is);
1546                         return (ENOMEM);
1547                 }
1548
1549                 is->is_sim = cam_sim_alloc(iscsi_action, NULL, "iscsi",
1550                     is, is->is_id /* unit */, &is->is_lock,
1551                     1, ic->ic_maxtags, is->is_devq);
1552                 if (is->is_sim == NULL) {
1553                         ISCSI_SESSION_UNLOCK(is);
1554                         ISCSI_SESSION_WARN(is, "failed to allocate SIM");
1555                         cam_simq_free(is->is_devq);
1556                         iscsi_session_terminate(is);
1557                         return (ENOMEM);
1558                 }
1559
1560                 if (xpt_bus_register(is->is_sim, NULL, 0) != 0) {
1561                         ISCSI_SESSION_UNLOCK(is);
1562                         ISCSI_SESSION_WARN(is, "failed to register bus");
1563                         iscsi_session_terminate(is);
1564                         return (ENOMEM);
1565                 }
1566
1567                 error = xpt_create_path(&is->is_path, /*periph*/NULL,
1568                     cam_sim_path(is->is_sim), CAM_TARGET_WILDCARD,
1569                     CAM_LUN_WILDCARD);
1570                 if (error != CAM_REQ_CMP) {
1571                         ISCSI_SESSION_UNLOCK(is);
1572                         ISCSI_SESSION_WARN(is, "failed to create path");
1573                         iscsi_session_terminate(is);
1574                         return (ENOMEM);
1575                 }
1576                 ISCSI_SESSION_UNLOCK(is);
1577         }
1578
1579         return (0);
1580 }
1581
1582 static int
1583 iscsi_ioctl_daemon_fail(struct iscsi_softc *sc,
1584     struct iscsi_daemon_fail *fail)
1585 {
1586         struct iscsi_session *is;
1587
1588         sx_slock(&sc->sc_lock);
1589
1590         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1591                 if (is->is_id == fail->idf_session_id)
1592                         break;
1593         }
1594         if (is == NULL) {
1595                 sx_sunlock(&sc->sc_lock);
1596                 return (ESRCH);
1597         }
1598         ISCSI_SESSION_LOCK(is);
1599         ISCSI_SESSION_DEBUG(is, "iscsid(8) failed: %s",
1600             fail->idf_reason);
1601         strlcpy(is->is_reason, fail->idf_reason, sizeof(is->is_reason));
1602         //is->is_waiting_for_iscsid = false;
1603         //is->is_login_phase = true;
1604         //iscsi_session_reconnect(is);
1605         ISCSI_SESSION_UNLOCK(is);
1606         sx_sunlock(&sc->sc_lock);
1607
1608         return (0);
1609 }
1610
1611 #ifdef ICL_KERNEL_PROXY
1612 static int
1613 iscsi_ioctl_daemon_connect(struct iscsi_softc *sc,
1614     struct iscsi_daemon_connect *idc)
1615 {
1616         struct iscsi_session *is;
1617         struct sockaddr *from_sa, *to_sa;
1618         int error;
1619
1620         sx_slock(&sc->sc_lock);
1621         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1622                 if (is->is_id == idc->idc_session_id)
1623                         break;
1624         }
1625         if (is == NULL) {
1626                 sx_sunlock(&sc->sc_lock);
1627                 return (ESRCH);
1628         }
1629         sx_sunlock(&sc->sc_lock);
1630
1631         if (idc->idc_from_addrlen > 0) {
1632                 error = getsockaddr(&from_sa, (void *)idc->idc_from_addr, idc->idc_from_addrlen);
1633                 if (error != 0) {
1634                         ISCSI_SESSION_WARN(is,
1635                             "getsockaddr failed with error %d", error);
1636                         return (error);
1637                 }
1638         } else {
1639                 from_sa = NULL;
1640         }
1641         error = getsockaddr(&to_sa, (void *)idc->idc_to_addr, idc->idc_to_addrlen);
1642         if (error != 0) {
1643                 ISCSI_SESSION_WARN(is, "getsockaddr failed with error %d",
1644                     error);
1645                 free(from_sa, M_SONAME);
1646                 return (error);
1647         }
1648
1649         ISCSI_SESSION_LOCK(is);
1650         is->is_statsn = 0;
1651         is->is_cmdsn = 0;
1652         is->is_expcmdsn = 0;
1653         is->is_maxcmdsn = 0;
1654         is->is_waiting_for_iscsid = false;
1655         is->is_login_phase = true;
1656         is->is_timeout = 0;
1657         ISCSI_SESSION_UNLOCK(is);
1658
1659         error = icl_conn_connect(is->is_conn, idc->idc_domain,
1660             idc->idc_socktype, idc->idc_protocol, from_sa, to_sa);
1661         free(from_sa, M_SONAME);
1662         free(to_sa, M_SONAME);
1663
1664         /*
1665          * Digests are always disabled during login phase.
1666          */
1667         is->is_conn->ic_header_crc32c = false;
1668         is->is_conn->ic_data_crc32c = false;
1669
1670         return (error);
1671 }
1672
1673 static int
1674 iscsi_ioctl_daemon_send(struct iscsi_softc *sc,
1675     struct iscsi_daemon_send *ids)
1676 {
1677         struct iscsi_session *is;
1678         struct icl_pdu *ip;
1679         size_t datalen;
1680         void *data;
1681         int error;
1682
1683         sx_slock(&sc->sc_lock);
1684         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1685                 if (is->is_id == ids->ids_session_id)
1686                         break;
1687         }
1688         if (is == NULL) {
1689                 sx_sunlock(&sc->sc_lock);
1690                 return (ESRCH);
1691         }
1692         sx_sunlock(&sc->sc_lock);
1693
1694         if (is->is_login_phase == false)
1695                 return (EBUSY);
1696
1697         if (is->is_terminating || is->is_reconnecting)
1698                 return (EIO);
1699
1700         datalen = ids->ids_data_segment_len;
1701         if (datalen > is->is_conn->ic_max_send_data_segment_length)
1702                 return (EINVAL);
1703         if (datalen > 0) {
1704                 data = malloc(datalen, M_ISCSI, M_WAITOK);
1705                 error = copyin(ids->ids_data_segment, data, datalen);
1706                 if (error != 0) {
1707                         free(data, M_ISCSI);
1708                         return (error);
1709                 }
1710         }
1711
1712         ip = icl_pdu_new(is->is_conn, M_WAITOK);
1713         memcpy(ip->ip_bhs, ids->ids_bhs, sizeof(*ip->ip_bhs));
1714         if (datalen > 0) {
1715                 error = icl_pdu_append_data(ip, data, datalen, M_WAITOK);
1716                 KASSERT(error == 0, ("icl_pdu_append_data(..., M_WAITOK) failed"));
1717                 free(data, M_ISCSI);
1718         }
1719         iscsi_pdu_queue(ip);
1720
1721         return (0);
1722 }
1723
1724 static int
1725 iscsi_ioctl_daemon_receive(struct iscsi_softc *sc,
1726     struct iscsi_daemon_receive *idr)
1727 {
1728         struct iscsi_session *is;
1729         struct icl_pdu *ip;
1730         void *data;
1731         int error;
1732
1733         sx_slock(&sc->sc_lock);
1734         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1735                 if (is->is_id == idr->idr_session_id)
1736                         break;
1737         }
1738         if (is == NULL) {
1739                 sx_sunlock(&sc->sc_lock);
1740                 return (ESRCH);
1741         }
1742         sx_sunlock(&sc->sc_lock);
1743
1744         if (is->is_login_phase == false)
1745                 return (EBUSY);
1746
1747         ISCSI_SESSION_LOCK(is);
1748         while (is->is_login_pdu == NULL &&
1749             is->is_terminating == false &&
1750             is->is_reconnecting == false) {
1751                 error = cv_wait_sig(&is->is_login_cv, &is->is_lock);
1752                 if (error != 0) {
1753                         ISCSI_SESSION_UNLOCK(is);
1754                         return (error);
1755                 }
1756         }
1757         if (is->is_terminating || is->is_reconnecting) {
1758                 ISCSI_SESSION_UNLOCK(is);
1759                 return (EIO);
1760         }
1761         ip = is->is_login_pdu;
1762         is->is_login_pdu = NULL;
1763         ISCSI_SESSION_UNLOCK(is);
1764
1765         if (ip->ip_data_len > idr->idr_data_segment_len) {
1766                 icl_pdu_free(ip);
1767                 return (EMSGSIZE);
1768         }
1769
1770         copyout(ip->ip_bhs, idr->idr_bhs, sizeof(*ip->ip_bhs));
1771         if (ip->ip_data_len > 0) {
1772                 data = malloc(ip->ip_data_len, M_ISCSI, M_WAITOK);
1773                 icl_pdu_get_data(ip, 0, data, ip->ip_data_len);
1774                 copyout(data, idr->idr_data_segment, ip->ip_data_len);
1775                 free(data, M_ISCSI);
1776         }
1777
1778         icl_pdu_free(ip);
1779
1780         return (0);
1781 }
1782 #endif /* ICL_KERNEL_PROXY */
1783
1784 static void
1785 iscsi_sanitize_session_conf(struct iscsi_session_conf *isc)
1786 {
1787         /*
1788          * Just make sure all the fields are null-terminated.
1789          *
1790          * XXX: This is not particularly secure.  We should
1791          *      create our own conf and then copy in relevant
1792          *      fields.
1793          */
1794         isc->isc_initiator[ISCSI_NAME_LEN - 1] = '\0';
1795         isc->isc_initiator_addr[ISCSI_ADDR_LEN - 1] = '\0';
1796         isc->isc_initiator_alias[ISCSI_ALIAS_LEN - 1] = '\0';
1797         isc->isc_target[ISCSI_NAME_LEN - 1] = '\0';
1798         isc->isc_target_addr[ISCSI_ADDR_LEN - 1] = '\0';
1799         isc->isc_user[ISCSI_NAME_LEN - 1] = '\0';
1800         isc->isc_secret[ISCSI_SECRET_LEN - 1] = '\0';
1801         isc->isc_mutual_user[ISCSI_NAME_LEN - 1] = '\0';
1802         isc->isc_mutual_secret[ISCSI_SECRET_LEN - 1] = '\0';
1803 }
1804
1805 static bool
1806 iscsi_valid_session_conf(const struct iscsi_session_conf *isc)
1807 {
1808
1809         if (isc->isc_initiator[0] == '\0') {
1810                 ISCSI_DEBUG("empty isc_initiator");
1811                 return (false);
1812         }
1813
1814         if (isc->isc_target_addr[0] == '\0') {
1815                 ISCSI_DEBUG("empty isc_target_addr");
1816                 return (false);
1817         }
1818
1819         if (isc->isc_discovery != 0 && isc->isc_target[0] != 0) {
1820                 ISCSI_DEBUG("non-empty isc_target for discovery session");
1821                 return (false);
1822         }
1823
1824         if (isc->isc_discovery == 0 && isc->isc_target[0] == 0) {
1825                 ISCSI_DEBUG("empty isc_target for non-discovery session");
1826                 return (false);
1827         }
1828
1829         return (true);
1830 }
1831
1832 static int
1833 iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
1834 {
1835         struct iscsi_session *is;
1836         const struct iscsi_session *is2;
1837         int error;
1838
1839         iscsi_sanitize_session_conf(&isa->isa_conf);
1840         if (iscsi_valid_session_conf(&isa->isa_conf) == false)
1841                 return (EINVAL);
1842
1843         is = malloc(sizeof(*is), M_ISCSI, M_ZERO | M_WAITOK);
1844         memcpy(&is->is_conf, &isa->isa_conf, sizeof(is->is_conf));
1845
1846         sx_xlock(&sc->sc_lock);
1847
1848         /*
1849          * Prevent duplicates.
1850          */
1851         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
1852                 if (!!is->is_conf.isc_discovery !=
1853                     !!is2->is_conf.isc_discovery)
1854                         continue;
1855
1856                 if (strcmp(is->is_conf.isc_target_addr,
1857                     is2->is_conf.isc_target_addr) != 0)
1858                         continue;
1859
1860                 if (is->is_conf.isc_discovery == 0 &&
1861                     strcmp(is->is_conf.isc_target,
1862                     is2->is_conf.isc_target) != 0)
1863                         continue;
1864
1865                 sx_xunlock(&sc->sc_lock);
1866                 free(is, M_ISCSI);
1867                 return (EBUSY);
1868         }
1869
1870         is->is_conn = icl_new_conn(is->is_conf.isc_offload,
1871             is->is_conf.isc_iser, "iscsi", &is->is_lock);
1872         if (is->is_conn == NULL) {
1873                 sx_xunlock(&sc->sc_lock);
1874                 free(is, M_ISCSI);
1875                 return (EINVAL);
1876         }
1877         is->is_conn->ic_receive = iscsi_receive_callback;
1878         is->is_conn->ic_error = iscsi_error_callback;
1879         is->is_conn->ic_prv0 = is;
1880         TAILQ_INIT(&is->is_outstanding);
1881         STAILQ_INIT(&is->is_postponed);
1882         mtx_init(&is->is_lock, "iscsi_lock", NULL, MTX_DEF);
1883         cv_init(&is->is_maintenance_cv, "iscsi_mt");
1884 #ifdef ICL_KERNEL_PROXY
1885         cv_init(&is->is_login_cv, "iscsi_login");
1886 #endif
1887
1888         /*
1889          * Set some default values, from RFC 3720, section 12.
1890          *
1891          * These values are updated by the handoff IOCTL, but are
1892          * needed prior to the handoff to support sending the ISER
1893          * login PDU.
1894          */
1895         is->is_conn->ic_max_recv_data_segment_length = 8192;
1896         is->is_conn->ic_max_send_data_segment_length = 8192;
1897         is->is_max_burst_length = 262144;
1898         is->is_first_burst_length = 65536;
1899
1900         is->is_softc = sc;
1901         sc->sc_last_session_id++;
1902         is->is_id = sc->sc_last_session_id;
1903         is->is_isid[0] = 0x80; /* RFC 3720, 10.12.5: 10b, "Random" ISID. */
1904         arc4rand(&is->is_isid[1], 5, 0);
1905         is->is_tsih = 0;
1906         callout_init(&is->is_callout, 1);
1907
1908         error = kthread_add(iscsi_maintenance_thread, is, NULL, NULL, 0, 0, "iscsimt");
1909         if (error != 0) {
1910                 ISCSI_SESSION_WARN(is, "kthread_add(9) failed with error %d", error);
1911                 sx_xunlock(&sc->sc_lock);
1912                 return (error);
1913         }
1914
1915         callout_reset(&is->is_callout, 1 * hz, iscsi_callout, is);
1916         TAILQ_INSERT_TAIL(&sc->sc_sessions, is, is_next);
1917
1918         ISCSI_SESSION_LOCK(is);
1919         /*
1920          * Don't notify iscsid(8) if the session is disabled and it's not
1921          * a discovery session,
1922          */
1923         if (is->is_conf.isc_enable == 0 && is->is_conf.isc_discovery == 0) {
1924                 ISCSI_SESSION_UNLOCK(is);
1925                 sx_xunlock(&sc->sc_lock);
1926                 return (0);
1927         }
1928
1929         is->is_waiting_for_iscsid = true;
1930         strlcpy(is->is_reason, "Waiting for iscsid(8)", sizeof(is->is_reason));
1931         ISCSI_SESSION_UNLOCK(is);
1932         cv_signal(&sc->sc_cv);
1933         sx_xunlock(&sc->sc_lock);
1934         return (0);
1935 }
1936
1937 static bool
1938 iscsi_session_conf_matches(unsigned int id1, const struct iscsi_session_conf *c1,
1939     unsigned int id2, const struct iscsi_session_conf *c2)
1940 {
1941
1942         if (id2 != 0 && id2 != id1)
1943                 return (false);
1944         if (c2->isc_target[0] != '\0' &&
1945             strcmp(c1->isc_target, c2->isc_target) != 0)
1946                 return (false);
1947         if (c2->isc_target_addr[0] != '\0' &&
1948             strcmp(c1->isc_target_addr, c2->isc_target_addr) != 0)
1949                 return (false);
1950         return (true);
1951 }
1952
1953 static int
1954 iscsi_ioctl_session_remove(struct iscsi_softc *sc,
1955     struct iscsi_session_remove *isr)
1956 {
1957         struct iscsi_session *is, *tmp;
1958         bool found = false;
1959
1960         iscsi_sanitize_session_conf(&isr->isr_conf);
1961
1962         sx_xlock(&sc->sc_lock);
1963         TAILQ_FOREACH_SAFE(is, &sc->sc_sessions, is_next, tmp) {
1964                 ISCSI_SESSION_LOCK(is);
1965                 if (iscsi_session_conf_matches(is->is_id, &is->is_conf,
1966                     isr->isr_session_id, &isr->isr_conf)) {
1967                         found = true;
1968                         iscsi_session_logout(is);
1969                         iscsi_session_terminate(is);
1970                 }
1971                 ISCSI_SESSION_UNLOCK(is);
1972         }
1973         sx_xunlock(&sc->sc_lock);
1974
1975         if (!found)
1976                 return (ESRCH);
1977
1978         return (0);
1979 }
1980
1981 static int
1982 iscsi_ioctl_session_list(struct iscsi_softc *sc, struct iscsi_session_list *isl)
1983 {
1984         int error;
1985         unsigned int i = 0;
1986         struct iscsi_session *is;
1987         struct iscsi_session_state iss;
1988
1989         sx_slock(&sc->sc_lock);
1990         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
1991                 if (i >= isl->isl_nentries) {
1992                         sx_sunlock(&sc->sc_lock);
1993                         return (EMSGSIZE);
1994                 }
1995                 memset(&iss, 0, sizeof(iss));
1996                 memcpy(&iss.iss_conf, &is->is_conf, sizeof(iss.iss_conf));
1997                 iss.iss_id = is->is_id;
1998                 strlcpy(iss.iss_target_alias, is->is_target_alias, sizeof(iss.iss_target_alias));
1999                 strlcpy(iss.iss_reason, is->is_reason, sizeof(iss.iss_reason));
2000                 strlcpy(iss.iss_offload, is->is_conn->ic_offload, sizeof(iss.iss_offload));
2001
2002                 if (is->is_conn->ic_header_crc32c)
2003                         iss.iss_header_digest = ISCSI_DIGEST_CRC32C;
2004                 else
2005                         iss.iss_header_digest = ISCSI_DIGEST_NONE;
2006
2007                 if (is->is_conn->ic_data_crc32c)
2008                         iss.iss_data_digest = ISCSI_DIGEST_CRC32C;
2009                 else
2010                         iss.iss_data_digest = ISCSI_DIGEST_NONE;
2011
2012                 iss.iss_max_send_data_segment_length =
2013                     is->is_conn->ic_max_send_data_segment_length;
2014                 iss.iss_max_recv_data_segment_length =
2015                     is->is_conn->ic_max_recv_data_segment_length;
2016                 iss.iss_max_burst_length = is->is_max_burst_length;
2017                 iss.iss_first_burst_length = is->is_first_burst_length;
2018                 iss.iss_immediate_data = is->is_immediate_data;
2019                 iss.iss_connected = is->is_connected;
2020
2021                 error = copyout(&iss, isl->isl_pstates + i, sizeof(iss));
2022                 if (error != 0) {
2023                         sx_sunlock(&sc->sc_lock);
2024                         return (error);
2025                 }
2026                 i++;
2027         }
2028         sx_sunlock(&sc->sc_lock);
2029
2030         isl->isl_nentries = i;
2031
2032         return (0);
2033 }
2034
2035 static int
2036 iscsi_ioctl_session_modify(struct iscsi_softc *sc,
2037     struct iscsi_session_modify *ism)
2038 {
2039         struct iscsi_session *is;
2040         const struct iscsi_session *is2;
2041
2042         iscsi_sanitize_session_conf(&ism->ism_conf);
2043         if (iscsi_valid_session_conf(&ism->ism_conf) == false)
2044                 return (EINVAL);
2045
2046         sx_xlock(&sc->sc_lock);
2047         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
2048                 ISCSI_SESSION_LOCK(is);
2049                 if (is->is_id == ism->ism_session_id) {
2050                         /* Note that the session remains locked. */
2051                         break;
2052                 }
2053                 ISCSI_SESSION_UNLOCK(is);
2054         }
2055         if (is == NULL) {
2056                 sx_xunlock(&sc->sc_lock);
2057                 return (ESRCH);
2058         }
2059
2060         /*
2061          * Prevent duplicates.
2062          */
2063         TAILQ_FOREACH(is2, &sc->sc_sessions, is_next) {
2064                 if (is == is2)
2065                         continue;
2066
2067                 if (!!ism->ism_conf.isc_discovery !=
2068                     !!is2->is_conf.isc_discovery)
2069                         continue;
2070
2071                 if (strcmp(ism->ism_conf.isc_target_addr,
2072                     is2->is_conf.isc_target_addr) != 0)
2073                         continue;
2074
2075                 if (ism->ism_conf.isc_discovery == 0 &&
2076                     strcmp(ism->ism_conf.isc_target,
2077                     is2->is_conf.isc_target) != 0)
2078                         continue;
2079
2080                 ISCSI_SESSION_UNLOCK(is);
2081                 sx_xunlock(&sc->sc_lock);
2082                 return (EBUSY);
2083         }
2084
2085         sx_xunlock(&sc->sc_lock);
2086
2087         memcpy(&is->is_conf, &ism->ism_conf, sizeof(is->is_conf));
2088         ISCSI_SESSION_UNLOCK(is);
2089
2090         iscsi_session_reconnect(is);
2091
2092         return (0);
2093 }
2094
2095 static int
2096 iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
2097     struct thread *td)
2098 {
2099         struct iscsi_softc *sc;
2100
2101         sc = dev->si_drv1;
2102
2103         switch (cmd) {
2104         case ISCSIDWAIT:
2105                 return (iscsi_ioctl_daemon_wait(sc,
2106                     (struct iscsi_daemon_request *)arg));
2107         case ISCSIDHANDOFF:
2108                 return (iscsi_ioctl_daemon_handoff(sc,
2109                     (struct iscsi_daemon_handoff *)arg));
2110         case ISCSIDFAIL:
2111                 return (iscsi_ioctl_daemon_fail(sc,
2112                     (struct iscsi_daemon_fail *)arg));
2113 #ifdef ICL_KERNEL_PROXY
2114         case ISCSIDCONNECT:
2115                 return (iscsi_ioctl_daemon_connect(sc,
2116                     (struct iscsi_daemon_connect *)arg));
2117         case ISCSIDSEND:
2118                 return (iscsi_ioctl_daemon_send(sc,
2119                     (struct iscsi_daemon_send *)arg));
2120         case ISCSIDRECEIVE:
2121                 return (iscsi_ioctl_daemon_receive(sc,
2122                     (struct iscsi_daemon_receive *)arg));
2123 #endif /* ICL_KERNEL_PROXY */
2124         case ISCSISADD:
2125                 return (iscsi_ioctl_session_add(sc,
2126                     (struct iscsi_session_add *)arg));
2127         case ISCSISREMOVE:
2128                 return (iscsi_ioctl_session_remove(sc,
2129                     (struct iscsi_session_remove *)arg));
2130         case ISCSISLIST:
2131                 return (iscsi_ioctl_session_list(sc,
2132                     (struct iscsi_session_list *)arg));
2133         case ISCSISMODIFY:
2134                 return (iscsi_ioctl_session_modify(sc,
2135                     (struct iscsi_session_modify *)arg));
2136         default:
2137                 return (EINVAL);
2138         }
2139 }
2140
2141 static struct iscsi_outstanding *
2142 iscsi_outstanding_find(struct iscsi_session *is, uint32_t initiator_task_tag)
2143 {
2144         struct iscsi_outstanding *io;
2145
2146         ISCSI_SESSION_LOCK_ASSERT(is);
2147
2148         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2149                 if (io->io_initiator_task_tag == initiator_task_tag)
2150                         return (io);
2151         }
2152         return (NULL);
2153 }
2154
2155 static struct iscsi_outstanding *
2156 iscsi_outstanding_find_ccb(struct iscsi_session *is, union ccb *ccb)
2157 {
2158         struct iscsi_outstanding *io;
2159
2160         ISCSI_SESSION_LOCK_ASSERT(is);
2161
2162         TAILQ_FOREACH(io, &is->is_outstanding, io_next) {
2163                 if (io->io_ccb == ccb)
2164                         return (io);
2165         }
2166         return (NULL);
2167 }
2168
2169 static struct iscsi_outstanding *
2170 iscsi_outstanding_add(struct iscsi_session *is, struct icl_pdu *request,
2171     union ccb *ccb, uint32_t *initiator_task_tagp)
2172 {
2173         struct iscsi_outstanding *io;
2174         int error;
2175
2176         ISCSI_SESSION_LOCK_ASSERT(is);
2177
2178         io = uma_zalloc(iscsi_outstanding_zone, M_NOWAIT | M_ZERO);
2179         if (io == NULL) {
2180                 ISCSI_SESSION_WARN(is, "failed to allocate %zd bytes",
2181                     sizeof(*io));
2182                 return (NULL);
2183         }
2184
2185         error = icl_conn_task_setup(is->is_conn, request, &ccb->csio,
2186             initiator_task_tagp, &io->io_icl_prv);
2187         if (error != 0) {
2188                 ISCSI_SESSION_WARN(is,
2189                     "icl_conn_task_setup() failed with error %d", error);
2190                 uma_zfree(iscsi_outstanding_zone, io);
2191                 return (NULL);
2192         }
2193
2194         KASSERT(iscsi_outstanding_find(is, *initiator_task_tagp) == NULL,
2195             ("initiator_task_tag 0x%x already added", *initiator_task_tagp));
2196
2197         io->io_initiator_task_tag = *initiator_task_tagp;
2198         io->io_ccb = ccb;
2199         TAILQ_INSERT_TAIL(&is->is_outstanding, io, io_next);
2200         return (io);
2201 }
2202
2203 static void
2204 iscsi_outstanding_remove(struct iscsi_session *is, struct iscsi_outstanding *io)
2205 {
2206
2207         ISCSI_SESSION_LOCK_ASSERT(is);
2208
2209         icl_conn_task_done(is->is_conn, io->io_icl_prv);
2210         TAILQ_REMOVE(&is->is_outstanding, io, io_next);
2211         uma_zfree(iscsi_outstanding_zone, io);
2212 }
2213
2214 static void
2215 iscsi_action_abort(struct iscsi_session *is, union ccb *ccb)
2216 {
2217         struct icl_pdu *request;
2218         struct iscsi_bhs_task_management_request *bhstmr;
2219         struct ccb_abort *cab = &ccb->cab;
2220         struct iscsi_outstanding *io, *aio;
2221         uint32_t initiator_task_tag;
2222
2223         ISCSI_SESSION_LOCK_ASSERT(is);
2224
2225 #if 0
2226         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2227 #else
2228         if (is->is_login_phase) {
2229                 ccb->ccb_h.status = CAM_REQ_ABORTED;
2230                 xpt_done(ccb);
2231                 return;
2232         }
2233 #endif
2234
2235         aio = iscsi_outstanding_find_ccb(is, cab->abort_ccb);
2236         if (aio == NULL) {
2237                 ccb->ccb_h.status = CAM_REQ_CMP;
2238                 xpt_done(ccb);
2239                 return;
2240         }
2241
2242         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2243         if (request == NULL) {
2244                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2245                 xpt_done(ccb);
2246                 return;
2247         }
2248
2249         initiator_task_tag = is->is_initiator_task_tag++;
2250         if (initiator_task_tag == 0xffffffff)
2251                 initiator_task_tag = is->is_initiator_task_tag++;
2252
2253         io = iscsi_outstanding_add(is, request, NULL, &initiator_task_tag);
2254         if (io == NULL) {
2255                 icl_pdu_free(request);
2256                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2257                 xpt_done(ccb);
2258                 return;
2259         }
2260         io->io_referenced_task_tag = aio->io_initiator_task_tag;
2261
2262         bhstmr = (struct iscsi_bhs_task_management_request *)request->ip_bhs;
2263         bhstmr->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_REQUEST;
2264         bhstmr->bhstmr_function = 0x80 | BHSTMR_FUNCTION_ABORT_TASK;
2265         bhstmr->bhstmr_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
2266         bhstmr->bhstmr_initiator_task_tag = initiator_task_tag;
2267         bhstmr->bhstmr_referenced_task_tag = aio->io_initiator_task_tag;
2268
2269         iscsi_pdu_queue_locked(request);
2270 }
2271
2272 static void
2273 iscsi_action_scsiio(struct iscsi_session *is, union ccb *ccb)
2274 {
2275         struct icl_pdu *request;
2276         struct iscsi_bhs_scsi_command *bhssc;
2277         struct ccb_scsiio *csio;
2278         struct iscsi_outstanding *io;
2279         size_t len;
2280         uint32_t initiator_task_tag;
2281         int error;
2282
2283         ISCSI_SESSION_LOCK_ASSERT(is);
2284
2285 #if 0
2286         KASSERT(is->is_login_phase == false, ("%s called during Login Phase", __func__));
2287 #else
2288         if (is->is_login_phase) {
2289                 ISCSI_SESSION_DEBUG(is, "called during login phase");
2290                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2291                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2292                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2293                 }
2294                 ccb->ccb_h.status = CAM_REQ_ABORTED | CAM_DEV_QFRZN;
2295                 xpt_done(ccb);
2296                 return;
2297         }
2298 #endif
2299
2300         request = icl_pdu_new(is->is_conn, M_NOWAIT);
2301         if (request == NULL) {
2302                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2303                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2304                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2305                 }
2306                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2307                 xpt_done(ccb);
2308                 return;
2309         }
2310
2311         initiator_task_tag = is->is_initiator_task_tag++;
2312         if (initiator_task_tag == 0xffffffff)
2313                 initiator_task_tag = is->is_initiator_task_tag++;
2314
2315         io = iscsi_outstanding_add(is, request, ccb, &initiator_task_tag);
2316         if (io == NULL) {
2317                 icl_pdu_free(request);
2318                 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2319                         xpt_freeze_devq(ccb->ccb_h.path, 1);
2320                         ISCSI_SESSION_DEBUG(is, "freezing devq");
2321                 }
2322                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2323                 xpt_done(ccb);
2324                 return;
2325         }
2326
2327         csio = &ccb->csio;
2328         bhssc = (struct iscsi_bhs_scsi_command *)request->ip_bhs;
2329         bhssc->bhssc_opcode = ISCSI_BHS_OPCODE_SCSI_COMMAND;
2330         bhssc->bhssc_flags |= BHSSC_FLAGS_F;
2331         switch (csio->ccb_h.flags & CAM_DIR_MASK) {
2332         case CAM_DIR_IN:
2333                 bhssc->bhssc_flags |= BHSSC_FLAGS_R;
2334                 break;
2335         case CAM_DIR_OUT:
2336                 bhssc->bhssc_flags |= BHSSC_FLAGS_W;
2337                 break;
2338         }
2339
2340         if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
2341                 switch (csio->tag_action) {
2342                 case MSG_HEAD_OF_Q_TAG:
2343                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_HOQ;
2344                         break;
2345                 case MSG_ORDERED_Q_TAG:
2346                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ORDERED;
2347                         break;
2348                 case MSG_ACA_TASK:
2349                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_ACA;
2350                         break;
2351                 case MSG_SIMPLE_Q_TAG:
2352                 default:
2353                         bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_SIMPLE;
2354                         break;
2355                 }
2356         } else
2357                 bhssc->bhssc_flags |= BHSSC_FLAGS_ATTR_UNTAGGED;
2358
2359         if (is->is_protocol_level >= 2) {
2360                 bhssc->bhssc_pri = (csio->priority << BHSSC_PRI_SHIFT) &
2361                     BHSSC_PRI_MASK;
2362         }
2363
2364         bhssc->bhssc_lun = htobe64(CAM_EXTLUN_BYTE_SWIZZLE(ccb->ccb_h.target_lun));
2365         bhssc->bhssc_initiator_task_tag = initiator_task_tag;
2366         bhssc->bhssc_expected_data_transfer_length = htonl(csio->dxfer_len);
2367         KASSERT(csio->cdb_len <= sizeof(bhssc->bhssc_cdb),
2368             ("unsupported CDB size %zd", (size_t)csio->cdb_len));
2369
2370         if (csio->ccb_h.flags & CAM_CDB_POINTER)
2371                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_ptr, csio->cdb_len);
2372         else
2373                 memcpy(&bhssc->bhssc_cdb, csio->cdb_io.cdb_bytes, csio->cdb_len);
2374
2375         if (is->is_immediate_data &&
2376             (csio->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
2377                 len = csio->dxfer_len;
2378                 //ISCSI_SESSION_DEBUG(is, "adding %zd of immediate data", len);
2379                 if (len > is->is_first_burst_length) {
2380                         ISCSI_SESSION_DEBUG(is, "len %zd -> %d", len, is->is_first_burst_length);
2381                         len = is->is_first_burst_length;
2382                 }
2383                 if (len > is->is_conn->ic_max_send_data_segment_length) {
2384                         ISCSI_SESSION_DEBUG(is, "len %zd -> %d", len,
2385                             is->is_conn->ic_max_send_data_segment_length);
2386                         len = is->is_conn->ic_max_send_data_segment_length;
2387                 }
2388
2389                 error = icl_pdu_append_data(request, csio->data_ptr, len, M_NOWAIT);
2390                 if (error != 0) {
2391                         iscsi_outstanding_remove(is, io);
2392                         icl_pdu_free(request);
2393                         if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
2394                                 xpt_freeze_devq(ccb->ccb_h.path, 1);
2395                                 ISCSI_SESSION_DEBUG(is, "freezing devq");
2396                         }
2397                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL | CAM_DEV_QFRZN;
2398                         xpt_done(ccb);
2399                         return;
2400                 }
2401         }
2402         iscsi_pdu_queue_locked(request);
2403 }
2404
2405 static void
2406 iscsi_action(struct cam_sim *sim, union ccb *ccb)
2407 {
2408         struct iscsi_session *is;
2409
2410         is = cam_sim_softc(sim);
2411
2412         ISCSI_SESSION_LOCK_ASSERT(is);
2413
2414         if (is->is_terminating ||
2415             (is->is_connected == false && fail_on_disconnection)) {
2416                 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2417                 xpt_done(ccb);
2418                 return;
2419         }
2420
2421         /*
2422          * Make sure CAM doesn't sneak in a CCB just after freezing the queue.
2423          */
2424         if (is->is_simq_frozen == true) {
2425                 ccb->ccb_h.status &= ~(CAM_SIM_QUEUED | CAM_STATUS_MASK);
2426                 ccb->ccb_h.status |= CAM_REQUEUE_REQ;
2427                 /* Don't freeze the devq - the SIM queue is already frozen. */
2428                 xpt_done(ccb);
2429                 return;
2430         }
2431
2432         switch (ccb->ccb_h.func_code) {
2433         case XPT_PATH_INQ:
2434         {
2435                 struct ccb_pathinq *cpi = &ccb->cpi;
2436
2437                 cpi->version_num = 1;
2438                 cpi->hba_inquiry = PI_TAG_ABLE;
2439                 cpi->target_sprt = 0;
2440                 cpi->hba_misc = PIM_EXTLUNS;
2441                 /*
2442                  * XXX: It shouldn't ever be NULL; this could be turned
2443                  *      into a KASSERT eventually.
2444                  */
2445                 if (is->is_conn == NULL)
2446                         ISCSI_WARN("NULL conn");
2447                 else if (is->is_conn->ic_unmapped)
2448                         cpi->hba_misc |= PIM_UNMAPPED;
2449                 cpi->hba_eng_cnt = 0;
2450                 cpi->max_target = 0;
2451                 /*
2452                  * Note that the variable below is only relevant for targets
2453                  * that don't claim compliance with anything above SPC2, which
2454                  * means they don't support REPORT_LUNS.
2455                  */
2456                 cpi->max_lun = 255;
2457                 cpi->initiator_id = ~0;
2458                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2459                 strlcpy(cpi->hba_vid, "iSCSI", HBA_IDLEN);
2460                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2461                 cpi->unit_number = cam_sim_unit(sim);
2462                 cpi->bus_id = cam_sim_bus(sim);
2463                 cpi->base_transfer_speed = 150000; /* XXX */
2464                 cpi->transport = XPORT_ISCSI;
2465                 cpi->transport_version = 0;
2466                 cpi->protocol = PROTO_SCSI;
2467                 cpi->protocol_version = SCSI_REV_SPC3;
2468                 cpi->maxio = maxphys;
2469                 cpi->ccb_h.status = CAM_REQ_CMP;
2470                 break;
2471         }
2472         case XPT_GET_TRAN_SETTINGS:
2473         {
2474                 struct ccb_trans_settings       *cts;
2475                 struct ccb_trans_settings_scsi  *scsi;
2476
2477                 cts = &ccb->cts;
2478                 scsi = &cts->proto_specific.scsi;
2479
2480                 cts->protocol = PROTO_SCSI;
2481                 cts->protocol_version = SCSI_REV_SPC3;
2482                 cts->transport = XPORT_ISCSI;
2483                 cts->transport_version = 0;
2484                 scsi->valid = CTS_SCSI_VALID_TQ;
2485                 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
2486                 cts->ccb_h.status = CAM_REQ_CMP;
2487                 break;
2488         }
2489         case XPT_CALC_GEOMETRY:
2490                 cam_calc_geometry(&ccb->ccg, /*extended*/1);
2491                 ccb->ccb_h.status = CAM_REQ_CMP;
2492                 break;
2493 #if 0
2494         /*
2495          * XXX: What's the point?
2496          */
2497         case XPT_RESET_BUS:
2498         case XPT_TERM_IO:
2499                 ISCSI_SESSION_DEBUG(is, "faking success for reset, abort, or term_io");
2500                 ccb->ccb_h.status = CAM_REQ_CMP;
2501                 break;
2502 #endif
2503         case XPT_ABORT:
2504                 iscsi_action_abort(is, ccb);
2505                 return;
2506         case XPT_SCSI_IO:
2507                 iscsi_action_scsiio(is, ccb);
2508                 return;
2509         default:
2510 #if 0
2511                 ISCSI_SESSION_DEBUG(is, "got unsupported code 0x%x", ccb->ccb_h.func_code);
2512 #endif
2513                 ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
2514                 break;
2515         }
2516         xpt_done(ccb);
2517 }
2518
2519 static void
2520 iscsi_terminate_sessions(struct iscsi_softc *sc)
2521 {
2522         struct iscsi_session *is;
2523
2524         sx_slock(&sc->sc_lock);
2525         TAILQ_FOREACH(is, &sc->sc_sessions, is_next)
2526                 iscsi_session_terminate(is);
2527         while(!TAILQ_EMPTY(&sc->sc_sessions)) {
2528                 ISCSI_DEBUG("waiting for sessions to terminate");
2529                 cv_wait(&sc->sc_cv, &sc->sc_lock);
2530         }
2531         ISCSI_DEBUG("all sessions terminated");
2532         sx_sunlock(&sc->sc_lock);
2533 }
2534
2535 static void
2536 iscsi_shutdown_pre(struct iscsi_softc *sc)
2537 {
2538         struct iscsi_session *is;
2539
2540         if (!fail_on_shutdown)
2541                 return;
2542
2543         /*
2544          * If we have any sessions waiting for reconnection, request
2545          * maintenance thread to fail them immediately instead of waiting
2546          * for reconnect timeout.
2547          *
2548          * This prevents LUNs with mounted filesystems that are supported
2549          * by disconnected iSCSI sessions from hanging, however it will
2550          * fail all queued BIOs.
2551          */
2552         ISCSI_DEBUG("forcing failing all disconnected sessions due to shutdown");
2553
2554         fail_on_disconnection = 1;
2555
2556         sx_slock(&sc->sc_lock);
2557         TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
2558                 ISCSI_SESSION_LOCK(is);
2559                 if (!is->is_connected) {
2560                         ISCSI_SESSION_DEBUG(is, "force failing disconnected session early");
2561                         iscsi_session_reconnect(is);
2562                 }
2563                 ISCSI_SESSION_UNLOCK(is);
2564         }
2565         sx_sunlock(&sc->sc_lock);
2566 }
2567
2568 static void
2569 iscsi_shutdown_post(struct iscsi_softc *sc)
2570 {
2571
2572         if (!KERNEL_PANICKED()) {
2573                 ISCSI_DEBUG("removing all sessions due to shutdown");
2574                 iscsi_terminate_sessions(sc);
2575         }
2576 }
2577
2578 static int
2579 iscsi_load(void)
2580 {
2581         int error;
2582
2583         sc = malloc(sizeof(*sc), M_ISCSI, M_ZERO | M_WAITOK);
2584         sx_init(&sc->sc_lock, "iscsi");
2585         TAILQ_INIT(&sc->sc_sessions);
2586         cv_init(&sc->sc_cv, "iscsi_cv");
2587
2588         iscsi_outstanding_zone = uma_zcreate("iscsi_outstanding",
2589             sizeof(struct iscsi_outstanding), NULL, NULL, NULL, NULL,
2590             UMA_ALIGN_PTR, 0);
2591
2592         error = make_dev_p(MAKEDEV_CHECKNAME, &sc->sc_cdev, &iscsi_cdevsw,
2593             NULL, UID_ROOT, GID_WHEEL, 0600, "iscsi");
2594         if (error != 0) {
2595                 ISCSI_WARN("failed to create device node, error %d", error);
2596                 return (error);
2597         }
2598         sc->sc_cdev->si_drv1 = sc;
2599
2600         sc->sc_shutdown_pre_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
2601             iscsi_shutdown_pre, sc, SHUTDOWN_PRI_FIRST);
2602         /*
2603          * shutdown_post_sync needs to run after filesystem shutdown and before
2604          * CAM shutdown - otherwise when rebooting with an iSCSI session that is
2605          * disconnected but has outstanding requests, dashutdown() will hang on
2606          * cam_periph_runccb().
2607          */
2608         sc->sc_shutdown_post_eh = EVENTHANDLER_REGISTER(shutdown_post_sync,
2609             iscsi_shutdown_post, sc, SHUTDOWN_PRI_DEFAULT - 1);
2610
2611         return (0);
2612 }
2613
2614 static int
2615 iscsi_unload(void)
2616 {
2617
2618         /* Awaken any threads asleep in iscsi_ioctl(). */
2619         sx_xlock(&sc->sc_lock);
2620         sc->sc_unloading = true;
2621         cv_signal(&sc->sc_cv);
2622         sx_xunlock(&sc->sc_lock);
2623
2624         if (sc->sc_cdev != NULL) {
2625                 ISCSI_DEBUG("removing device node");
2626                 destroy_dev(sc->sc_cdev);
2627                 ISCSI_DEBUG("device node removed");
2628         }
2629
2630         if (sc->sc_shutdown_pre_eh != NULL)
2631                 EVENTHANDLER_DEREGISTER(shutdown_pre_sync, sc->sc_shutdown_pre_eh);
2632         if (sc->sc_shutdown_post_eh != NULL)
2633                 EVENTHANDLER_DEREGISTER(shutdown_post_sync, sc->sc_shutdown_post_eh);
2634
2635         iscsi_terminate_sessions(sc);
2636
2637         uma_zdestroy(iscsi_outstanding_zone);
2638         sx_destroy(&sc->sc_lock);
2639         cv_destroy(&sc->sc_cv);
2640         free(sc, M_ISCSI);
2641         return (0);
2642 }
2643
2644 static int
2645 iscsi_quiesce(void)
2646 {
2647         sx_slock(&sc->sc_lock);
2648         if (!TAILQ_EMPTY(&sc->sc_sessions)) {
2649                 sx_sunlock(&sc->sc_lock);
2650                 return (EBUSY);
2651         }
2652         sx_sunlock(&sc->sc_lock);
2653         return (0);
2654 }
2655
2656 static int
2657 iscsi_modevent(module_t mod, int what, void *arg)
2658 {
2659         int error;
2660
2661         switch (what) {
2662         case MOD_LOAD:
2663                 error = iscsi_load();
2664                 break;
2665         case MOD_UNLOAD:
2666                 error = iscsi_unload();
2667                 break;
2668         case MOD_QUIESCE:
2669                 error = iscsi_quiesce();
2670                 break;
2671         default:
2672                 error = EINVAL;
2673                 break;
2674         }
2675         return (error);
2676 }
2677
2678 moduledata_t iscsi_data = {
2679         "iscsi",
2680         iscsi_modevent,
2681         0
2682 };
2683
2684 DECLARE_MODULE(iscsi, iscsi_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
2685 MODULE_DEPEND(iscsi, cam, 1, 1, 1);
2686 MODULE_DEPEND(iscsi, icl, 1, 1, 1);