sys/dev/iscsi/icl_soft.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2012 The FreeBSD Foundation
   5  *
   6  * This software was developed by Edward Tomasz Napierala under sponsorship
   7  * from the FreeBSD Foundation.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice, this list of conditions and the following disclaimer.
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  *    notice, this list of conditions and the following disclaimer in the
  16  *    documentation and/or other materials provided with the distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  19  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  28  * SUCH DAMAGE.
  29  *
  30  */
  31
  32 /*
  33  * Software implementation of iSCSI Common Layer kobj(9) interface.
  34  */
  35
  36 #include <sys/cdefs.h>
  37 __FBSDID("$FreeBSD$");
  38
  39 #include <sys/param.h>
  40 #include <sys/capsicum.h>
  41 #include <sys/condvar.h>
  42 #include <sys/conf.h>
  43 #include <sys/gsb_crc32.h>
  44 #include <sys/file.h>
  45 #include <sys/kernel.h>
  46 #include <sys/kthread.h>
  47 #include <sys/lock.h>
  48 #include <sys/mbuf.h>
  49 #include <sys/mutex.h>
  50 #include <sys/module.h>
  51 #include <sys/protosw.h>
  52 #include <sys/socket.h>
  53 #include <sys/socketvar.h>
  54 #include <sys/sysctl.h>
  55 #include <sys/systm.h>
  56 #include <sys/sx.h>
  57 #include <sys/uio.h>
  58 #include <vm/uma.h>
  59 #include <netinet/in.h>
  60 #include <netinet/tcp.h>
  61
  62 #include <dev/iscsi/icl.h>
  63 #include <dev/iscsi/iscsi_proto.h>
  64 #include <icl_conn_if.h>
  65
  66 struct icl_soft_pdu {
  67         struct icl_pdu   ip;
  68
  69         /* soft specific stuff goes here. */
  70         u_int            ref_cnt;
  71         icl_pdu_cb       cb;
  72         int              error;
  73 };
  74
  75 SYSCTL_NODE(_kern_icl, OID_AUTO, soft, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
  76     "Software iSCSI");
  77 static int coalesce = 1;
  78 SYSCTL_INT(_kern_icl_soft, OID_AUTO, coalesce, CTLFLAG_RWTUN,
  79     &coalesce, 0, "Try to coalesce PDUs before sending");
  80 static int partial_receive_len = 256 * 1024;
  81 SYSCTL_INT(_kern_icl_soft, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN,
  82     &partial_receive_len, 0, "Minimum read size for partially received "
  83     "data segment");
  84 static int max_data_segment_length = 256 * 1024;
  85 SYSCTL_INT(_kern_icl_soft, OID_AUTO, max_data_segment_length, CTLFLAG_RWTUN,
  86     &max_data_segment_length, 0, "Maximum data segment length");
  87 static int first_burst_length = 1024 * 1024;
  88 SYSCTL_INT(_kern_icl_soft, OID_AUTO, first_burst_length, CTLFLAG_RWTUN,
  89     &first_burst_length, 0, "First burst length");
  90 static int max_burst_length = 1024 * 1024;
  91 SYSCTL_INT(_kern_icl_soft, OID_AUTO, max_burst_length, CTLFLAG_RWTUN,
  92     &max_burst_length, 0, "Maximum burst length");
  93 static int sendspace = 1536 * 1024;
  94 SYSCTL_INT(_kern_icl_soft, OID_AUTO, sendspace, CTLFLAG_RWTUN,
  95     &sendspace, 0, "Default send socket buffer size");
  96 static int recvspace = 1536 * 1024;
  97 SYSCTL_INT(_kern_icl_soft, OID_AUTO, recvspace, CTLFLAG_RWTUN,
  98     &recvspace, 0, "Default receive socket buffer size");
  99
 100 static MALLOC_DEFINE(M_ICL_SOFT, "icl_soft", "iSCSI software backend");
 101 static uma_zone_t icl_soft_pdu_zone;
 102
 103 static volatile u_int   icl_ncons;
 104
 105 #define ICL_CONN_LOCK(X)                mtx_lock(X->ic_lock)
 106 #define ICL_CONN_UNLOCK(X)              mtx_unlock(X->ic_lock)
 107 #define ICL_CONN_LOCK_ASSERT(X)         mtx_assert(X->ic_lock, MA_OWNED)
 108 #define ICL_CONN_LOCK_ASSERT_NOT(X)     mtx_assert(X->ic_lock, MA_NOTOWNED)
 109
 110 STAILQ_HEAD(icl_pdu_stailq, icl_pdu);
 111
 112 static icl_conn_new_pdu_t       icl_soft_conn_new_pdu;
 113 static icl_conn_pdu_free_t      icl_soft_conn_pdu_free;
 114 static icl_conn_pdu_data_segment_length_t
 115                                     icl_soft_conn_pdu_data_segment_length;
 116 static icl_conn_pdu_append_data_t       icl_soft_conn_pdu_append_data;
 117 static icl_conn_pdu_get_data_t  icl_soft_conn_pdu_get_data;
 118 static icl_conn_pdu_queue_t     icl_soft_conn_pdu_queue;
 119 static icl_conn_pdu_queue_cb_t  icl_soft_conn_pdu_queue_cb;
 120 static icl_conn_handoff_t       icl_soft_conn_handoff;
 121 static icl_conn_free_t          icl_soft_conn_free;
 122 static icl_conn_close_t         icl_soft_conn_close;
 123 static icl_conn_task_setup_t    icl_soft_conn_task_setup;
 124 static icl_conn_task_done_t     icl_soft_conn_task_done;
 125 static icl_conn_transfer_setup_t        icl_soft_conn_transfer_setup;
 126 static icl_conn_transfer_done_t icl_soft_conn_transfer_done;
 127 #ifdef ICL_KERNEL_PROXY
 128 static icl_conn_connect_t       icl_soft_conn_connect;
 129 #endif
 130
 131 static kobj_method_t icl_soft_methods[] = {
 132         KOBJMETHOD(icl_conn_new_pdu, icl_soft_conn_new_pdu),
 133         KOBJMETHOD(icl_conn_pdu_free, icl_soft_conn_pdu_free),
 134         KOBJMETHOD(icl_conn_pdu_data_segment_length,
 135             icl_soft_conn_pdu_data_segment_length),
 136         KOBJMETHOD(icl_conn_pdu_append_data, icl_soft_conn_pdu_append_data),
 137         KOBJMETHOD(icl_conn_pdu_get_data, icl_soft_conn_pdu_get_data),
 138         KOBJMETHOD(icl_conn_pdu_queue, icl_soft_conn_pdu_queue),
 139         KOBJMETHOD(icl_conn_pdu_queue_cb, icl_soft_conn_pdu_queue_cb),
 140         KOBJMETHOD(icl_conn_handoff, icl_soft_conn_handoff),
 141         KOBJMETHOD(icl_conn_free, icl_soft_conn_free),
 142         KOBJMETHOD(icl_conn_close, icl_soft_conn_close),
 143         KOBJMETHOD(icl_conn_task_setup, icl_soft_conn_task_setup),
 144         KOBJMETHOD(icl_conn_task_done, icl_soft_conn_task_done),
 145         KOBJMETHOD(icl_conn_transfer_setup, icl_soft_conn_transfer_setup),
 146         KOBJMETHOD(icl_conn_transfer_done, icl_soft_conn_transfer_done),
 147 #ifdef ICL_KERNEL_PROXY
 148         KOBJMETHOD(icl_conn_connect, icl_soft_conn_connect),
 149 #endif
 150         { 0, 0 }
 151 };
 152
 153 DEFINE_CLASS(icl_soft, icl_soft_methods, sizeof(struct icl_conn));
 154
 155 static void
 156 icl_conn_fail(struct icl_conn *ic)
 157 {
 158         if (ic->ic_socket == NULL)
 159                 return;
 160
 161         /*
 162          * XXX
 163          */
 164         ic->ic_socket->so_error = EDOOFUS;
 165         (ic->ic_error)(ic);
 166 }
 167
 168 static void
 169 icl_soft_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
 170 {
 171         struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip;
 172
 173         KASSERT(isp->ref_cnt == 0, ("freeing active PDU"));
 174         m_freem(ip->ip_bhs_mbuf);
 175         m_freem(ip->ip_ahs_mbuf);
 176         m_freem(ip->ip_data_mbuf);
 177         uma_zfree(icl_soft_pdu_zone, isp);
 178 #ifdef DIAGNOSTIC
 179         refcount_release(&ic->ic_outstanding_pdus);
 180 #endif
 181 }
 182
 183 static void
 184 icl_soft_pdu_call_cb(struct icl_pdu *ip)
 185 {
 186         struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip;
 187
 188         if (isp->cb != NULL)
 189                 isp->cb(ip, isp->error);
 190 #ifdef DIAGNOSTIC
 191         refcount_release(&ip->ip_conn->ic_outstanding_pdus);
 192 #endif
 193         uma_zfree(icl_soft_pdu_zone, isp);
 194 }
 195
 196 static void
 197 icl_soft_pdu_done(struct icl_pdu *ip, int error)
 198 {
 199         struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip;
 200
 201         if (error != 0)
 202                 isp->error = error;
 203
 204         m_freem(ip->ip_bhs_mbuf);
 205         ip->ip_bhs_mbuf = NULL;
 206         m_freem(ip->ip_ahs_mbuf);
 207         ip->ip_ahs_mbuf = NULL;
 208         m_freem(ip->ip_data_mbuf);
 209         ip->ip_data_mbuf = NULL;
 210
 211         if (atomic_fetchadd_int(&isp->ref_cnt, -1) == 1)
 212                 icl_soft_pdu_call_cb(ip);
 213 }
 214
 215 static void
 216 icl_soft_mbuf_done(struct mbuf *mb)
 217 {
 218         struct icl_soft_pdu *isp = (struct icl_soft_pdu *)mb->m_ext.ext_arg1;
 219
 220         icl_soft_pdu_call_cb(&isp->ip);
 221 }
 222
 223 /*
 224  * Allocate icl_pdu with empty BHS to fill up by the caller.
 225  */
 226 struct icl_pdu *
 227 icl_soft_conn_new_pdu(struct icl_conn *ic, int flags)
 228 {
 229         struct icl_soft_pdu *isp;
 230         struct icl_pdu *ip;
 231
 232 #ifdef DIAGNOSTIC
 233         refcount_acquire(&ic->ic_outstanding_pdus);
 234 #endif
 235         isp = uma_zalloc(icl_soft_pdu_zone, flags | M_ZERO);
 236         if (isp == NULL) {
 237                 ICL_WARN("failed to allocate soft PDU");
 238 #ifdef DIAGNOSTIC
 239                 refcount_release(&ic->ic_outstanding_pdus);
 240 #endif
 241                 return (NULL);
 242         }
 243         ip = &isp->ip;
 244         ip->ip_conn = ic;
 245
 246         CTASSERT(sizeof(struct iscsi_bhs) <= MHLEN);
 247         ip->ip_bhs_mbuf = m_gethdr(flags, MT_DATA);
 248         if (ip->ip_bhs_mbuf == NULL) {
 249                 ICL_WARN("failed to allocate BHS mbuf");
 250                 icl_soft_conn_pdu_free(ic, ip);
 251                 return (NULL);
 252         }
 253         ip->ip_bhs = mtod(ip->ip_bhs_mbuf, struct iscsi_bhs *);
 254         memset(ip->ip_bhs, 0, sizeof(struct iscsi_bhs));
 255         ip->ip_bhs_mbuf->m_len = sizeof(struct iscsi_bhs);
 256
 257         return (ip);
 258 }
 259
 260 static int
 261 icl_pdu_ahs_length(const struct icl_pdu *request)
 262 {
 263
 264         return (request->ip_bhs->bhs_total_ahs_len * 4);
 265 }
 266
 267 static size_t
 268 icl_pdu_data_segment_length(const struct icl_pdu *request)
 269 {
 270         uint32_t len = 0;
 271
 272         len += request->ip_bhs->bhs_data_segment_len[0];
 273         len <<= 8;
 274         len += request->ip_bhs->bhs_data_segment_len[1];
 275         len <<= 8;
 276         len += request->ip_bhs->bhs_data_segment_len[2];
 277
 278         return (len);
 279 }
 280
 281 size_t
 282 icl_soft_conn_pdu_data_segment_length(struct icl_conn *ic,
 283     const struct icl_pdu *request)
 284 {
 285
 286         return (icl_pdu_data_segment_length(request));
 287 }
 288
 289 static void
 290 icl_pdu_set_data_segment_length(struct icl_pdu *response, uint32_t len)
 291 {
 292
 293         response->ip_bhs->bhs_data_segment_len[2] = len;
 294         response->ip_bhs->bhs_data_segment_len[1] = len >> 8;
 295         response->ip_bhs->bhs_data_segment_len[0] = len >> 16;
 296 }
 297
 298 static size_t
 299 icl_pdu_padding(const struct icl_pdu *ip)
 300 {
 301
 302         if ((ip->ip_data_len % 4) != 0)
 303                 return (4 - (ip->ip_data_len % 4));
 304
 305         return (0);
 306 }
 307
 308 static size_t
 309 icl_pdu_size(const struct icl_pdu *response)
 310 {
 311         size_t len;
 312
 313         KASSERT(response->ip_ahs_len == 0, ("responding with AHS"));
 314
 315         len = sizeof(struct iscsi_bhs) + response->ip_data_len +
 316             icl_pdu_padding(response);
 317         if (response->ip_conn->ic_header_crc32c)
 318                 len += ISCSI_HEADER_DIGEST_SIZE;
 319         if (response->ip_data_len != 0 && response->ip_conn->ic_data_crc32c)
 320                 len += ISCSI_DATA_DIGEST_SIZE;
 321
 322         return (len);
 323 }
 324
 325 static void
 326 icl_soft_receive_buf(struct mbuf **r, size_t *rs, void *buf, size_t s)
 327 {
 328
 329         m_copydata(*r, 0, s, buf);
 330         m_adj(*r, s);
 331         while ((*r) != NULL && (*r)->m_len == 0)
 332                 *r = m_free(*r);
 333         *rs -= s;
 334 }
 335
 336 static void
 337 icl_pdu_receive_ahs(struct icl_pdu *request, struct mbuf **r, size_t *rs)
 338 {
 339
 340         request->ip_ahs_len = icl_pdu_ahs_length(request);
 341         if (request->ip_ahs_len == 0)
 342                 return;
 343
 344         request->ip_ahs_mbuf = *r;
 345         *r = m_split(request->ip_ahs_mbuf, request->ip_ahs_len, M_WAITOK);
 346         *rs -= request->ip_ahs_len;
 347 }
 348
 349 static uint32_t
 350 icl_mbuf_to_crc32c(const struct mbuf *m0)
 351 {
 352         uint32_t digest = 0xffffffff;
 353         const struct mbuf *m;
 354
 355         for (m = m0; m != NULL; m = m->m_next)
 356                 digest = calculate_crc32c(digest,
 357                     mtod(m, const void *), m->m_len);
 358
 359         digest = digest ^ 0xffffffff;
 360
 361         return (digest);
 362 }
 363
 364 static int
 365 icl_pdu_check_header_digest(struct icl_pdu *request, struct mbuf **r, size_t *rs)
 366 {
 367         uint32_t received_digest, valid_digest;
 368
 369         if (request->ip_conn->ic_header_crc32c == false)
 370                 return (0);
 371
 372         CTASSERT(sizeof(received_digest) == ISCSI_HEADER_DIGEST_SIZE);
 373         icl_soft_receive_buf(r, rs, &received_digest, ISCSI_HEADER_DIGEST_SIZE);
 374
 375         /* Temporary attach AHS to BHS to calculate header digest. */
 376         request->ip_bhs_mbuf->m_next = request->ip_ahs_mbuf;
 377         valid_digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf);
 378         request->ip_bhs_mbuf->m_next = NULL;
 379         if (received_digest != valid_digest) {
 380                 ICL_WARN("header digest check failed; got 0x%x, "
 381                     "should be 0x%x", received_digest, valid_digest);
 382                 return (-1);
 383         }
 384
 385         return (0);
 386 }
 387
 388 /*
 389  * Return the number of bytes that should be waiting in the receive socket
 390  * before icl_pdu_receive_data_segment() gets called.
 391  */
 392 static size_t
 393 icl_pdu_data_segment_receive_len(const struct icl_pdu *request)
 394 {
 395         size_t len;
 396
 397         len = icl_pdu_data_segment_length(request);
 398         if (len == 0)
 399                 return (0);
 400
 401         /*
 402          * Account for the parts of data segment already read from
 403          * the socket buffer.
 404          */
 405         KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len"));
 406         len -= request->ip_data_len;
 407
 408         /*
 409          * Don't always wait for the full data segment to be delivered
 410          * to the socket; this might badly affect performance due to
 411          * TCP window scaling.
 412          */
 413         if (len > partial_receive_len) {
 414 #if 0
 415                 ICL_DEBUG("need %zd bytes of data, limiting to %zd",
 416                     len, partial_receive_len));
 417 #endif
 418                 len = partial_receive_len;
 419
 420                 return (len);
 421         }
 422
 423         /*
 424          * Account for padding.  Note that due to the way code is written,
 425          * the icl_pdu_receive_data_segment() must always receive padding
 426          * along with the last part of data segment, because it would be
 427          * impossible to tell whether we've already received the full data
 428          * segment including padding, or without it.
 429          */
 430         if ((len % 4) != 0)
 431                 len += 4 - (len % 4);
 432
 433 #if 0
 434         ICL_DEBUG("need %zd bytes of data", len));
 435 #endif
 436
 437         return (len);
 438 }
 439
 440 static int
 441 icl_pdu_receive_data_segment(struct icl_pdu *request, struct mbuf **r,
 442     size_t *rs, bool *more_neededp)
 443 {
 444         struct icl_conn *ic;
 445         size_t len, padding = 0;
 446         struct mbuf *m;
 447
 448         ic = request->ip_conn;
 449
 450         *more_neededp = false;
 451         ic->ic_receive_len = 0;
 452
 453         len = icl_pdu_data_segment_length(request);
 454         if (len == 0)
 455                 return (0);
 456
 457         if ((len % 4) != 0)
 458                 padding = 4 - (len % 4);
 459
 460         /*
 461          * Account for already received parts of data segment.
 462          */
 463         KASSERT(len > request->ip_data_len, ("len <= request->ip_data_len"));
 464         len -= request->ip_data_len;
 465
 466         if (len + padding > *rs) {
 467                 /*
 468                  * Not enough data in the socket buffer.  Receive as much
 469                  * as we can.  Don't receive padding, since, obviously, it's
 470                  * not the end of data segment yet.
 471                  */
 472 #if 0
 473                 ICL_DEBUG("limited from %zd to %zd",
 474                     len + padding, *rs - padding));
 475 #endif
 476                 len = *rs - padding;
 477                 *more_neededp = true;
 478                 padding = 0;
 479         }
 480
 481         /*
 482          * Must not try to receive padding without at least one byte
 483          * of actual data segment.
 484          */
 485         if (len > 0) {
 486                 m = *r;
 487                 *r = m_split(m, len + padding, M_WAITOK);
 488                 *rs -= len + padding;
 489
 490                 if (request->ip_data_mbuf == NULL)
 491                         request->ip_data_mbuf = m;
 492                 else
 493                         m_cat(request->ip_data_mbuf, m);
 494
 495                 request->ip_data_len += len;
 496         } else
 497                 ICL_DEBUG("len 0");
 498
 499         if (*more_neededp)
 500                 ic->ic_receive_len =
 501                     icl_pdu_data_segment_receive_len(request);
 502
 503         return (0);
 504 }
 505
 506 static int
 507 icl_pdu_check_data_digest(struct icl_pdu *request, struct mbuf **r, size_t *rs)
 508 {
 509         uint32_t received_digest, valid_digest;
 510
 511         if (request->ip_conn->ic_data_crc32c == false)
 512                 return (0);
 513
 514         if (request->ip_data_len == 0)
 515                 return (0);
 516
 517         CTASSERT(sizeof(received_digest) == ISCSI_DATA_DIGEST_SIZE);
 518         icl_soft_receive_buf(r, rs, &received_digest, ISCSI_DATA_DIGEST_SIZE);
 519
 520         /*
 521          * Note that ip_data_mbuf also contains padding; since digest
 522          * calculation is supposed to include that, we iterate over
 523          * the entire ip_data_mbuf chain, not just ip_data_len bytes of it.
 524          */
 525         valid_digest = icl_mbuf_to_crc32c(request->ip_data_mbuf);
 526         if (received_digest != valid_digest) {
 527                 ICL_WARN("data digest check failed; got 0x%x, "
 528                     "should be 0x%x", received_digest, valid_digest);
 529                 return (-1);
 530         }
 531
 532         return (0);
 533 }
 534
 535 /*
 536  * Somewhat contrary to the name, this attempts to receive only one
 537  * "part" of PDU at a time; call it repeatedly until it returns non-NULL.
 538  */
 539 static struct icl_pdu *
 540 icl_conn_receive_pdu(struct icl_conn *ic, struct mbuf **r, size_t *rs)
 541 {
 542         struct icl_pdu *request;
 543         size_t len;
 544         int error = 0;
 545         bool more_needed;
 546
 547         if (ic->ic_receive_state == ICL_CONN_STATE_BHS) {
 548                 KASSERT(ic->ic_receive_pdu == NULL,
 549                     ("ic->ic_receive_pdu != NULL"));
 550                 request = icl_soft_conn_new_pdu(ic, M_NOWAIT);
 551                 if (request == NULL) {
 552                         ICL_DEBUG("failed to allocate PDU; "
 553                             "dropping connection");
 554                         icl_conn_fail(ic);
 555                         return (NULL);
 556                 }
 557                 ic->ic_receive_pdu = request;
 558         } else {
 559                 KASSERT(ic->ic_receive_pdu != NULL,
 560                     ("ic->ic_receive_pdu == NULL"));
 561                 request = ic->ic_receive_pdu;
 562         }
 563
 564         switch (ic->ic_receive_state) {
 565         case ICL_CONN_STATE_BHS:
 566                 //ICL_DEBUG("receiving BHS");
 567                 icl_soft_receive_buf(r, rs, request->ip_bhs,
 568                     sizeof(struct iscsi_bhs));
 569
 570                 /*
 571                  * We don't enforce any limit for AHS length;
 572                  * its length is stored in 8 bit field.
 573                  */
 574
 575                 len = icl_pdu_data_segment_length(request);
 576                 if (len > ic->ic_max_recv_data_segment_length) {
 577                         ICL_WARN("received data segment "
 578                             "length %zd is larger than negotiated; "
 579                             "dropping connection", len);
 580                         error = EINVAL;
 581                         break;
 582                 }
 583
 584                 ic->ic_receive_state = ICL_CONN_STATE_AHS;
 585                 ic->ic_receive_len = icl_pdu_ahs_length(request);
 586                 break;
 587
 588         case ICL_CONN_STATE_AHS:
 589                 //ICL_DEBUG("receiving AHS");
 590                 icl_pdu_receive_ahs(request, r, rs);
 591                 ic->ic_receive_state = ICL_CONN_STATE_HEADER_DIGEST;
 592                 if (ic->ic_header_crc32c == false)
 593                         ic->ic_receive_len = 0;
 594                 else
 595                         ic->ic_receive_len = ISCSI_HEADER_DIGEST_SIZE;
 596                 break;
 597
 598         case ICL_CONN_STATE_HEADER_DIGEST:
 599                 //ICL_DEBUG("receiving header digest");
 600                 error = icl_pdu_check_header_digest(request, r, rs);
 601                 if (error != 0) {
 602                         ICL_DEBUG("header digest failed; "
 603                             "dropping connection");
 604                         break;
 605                 }
 606
 607                 ic->ic_receive_state = ICL_CONN_STATE_DATA;
 608                 ic->ic_receive_len =
 609                     icl_pdu_data_segment_receive_len(request);
 610                 break;
 611
 612         case ICL_CONN_STATE_DATA:
 613                 //ICL_DEBUG("receiving data segment");
 614                 error = icl_pdu_receive_data_segment(request, r, rs,
 615                     &more_needed);
 616                 if (error != 0) {
 617                         ICL_DEBUG("failed to receive data segment;"
 618                             "dropping connection");
 619                         break;
 620                 }
 621
 622                 if (more_needed)
 623                         break;
 624
 625                 ic->ic_receive_state = ICL_CONN_STATE_DATA_DIGEST;
 626                 if (request->ip_data_len == 0 || ic->ic_data_crc32c == false)
 627                         ic->ic_receive_len = 0;
 628                 else
 629                         ic->ic_receive_len = ISCSI_DATA_DIGEST_SIZE;
 630                 break;
 631
 632         case ICL_CONN_STATE_DATA_DIGEST:
 633                 //ICL_DEBUG("receiving data digest");
 634                 error = icl_pdu_check_data_digest(request, r, rs);
 635                 if (error != 0) {
 636                         ICL_DEBUG("data digest failed; "
 637                             "dropping connection");
 638                         break;
 639                 }
 640
 641                 /*
 642                  * We've received complete PDU; reset the receive state machine
 643                  * and return the PDU.
 644                  */
 645                 ic->ic_receive_state = ICL_CONN_STATE_BHS;
 646                 ic->ic_receive_len = sizeof(struct iscsi_bhs);
 647                 ic->ic_receive_pdu = NULL;
 648                 return (request);
 649
 650         default:
 651                 panic("invalid ic_receive_state %d\n", ic->ic_receive_state);
 652         }
 653
 654         if (error != 0) {
 655                 /*
 656                  * Don't free the PDU; it's pointed to by ic->ic_receive_pdu
 657                  * and will get freed in icl_soft_conn_close().
 658                  */
 659                 icl_conn_fail(ic);
 660         }
 661
 662         return (NULL);
 663 }
 664
 665 static void
 666 icl_conn_receive_pdus(struct icl_conn *ic, struct mbuf **r, size_t *rs)
 667 {
 668         struct icl_pdu *response;
 669
 670         for (;;) {
 671                 if (ic->ic_disconnecting)
 672                         return;
 673
 674                 /*
 675                  * Loop until we have a complete PDU or there is not enough
 676                  * data in the socket buffer.
 677                  */
 678                 if (*rs < ic->ic_receive_len) {
 679 #if 0
 680                         ICL_DEBUG("not enough data; have %zd, need %zd",
 681                             *rs, ic->ic_receive_len);
 682 #endif
 683                         return;
 684                 }
 685
 686                 response = icl_conn_receive_pdu(ic, r, rs);
 687                 if (response == NULL)
 688                         continue;
 689
 690                 if (response->ip_ahs_len > 0) {
 691                         ICL_WARN("received PDU with unsupported "
 692                             "AHS; opcode 0x%x; dropping connection",
 693                             response->ip_bhs->bhs_opcode);
 694                         icl_soft_conn_pdu_free(ic, response);
 695                         icl_conn_fail(ic);
 696                         return;
 697                 }
 698
 699                 (ic->ic_receive)(response);
 700         }
 701 }
 702
 703 static void
 704 icl_receive_thread(void *arg)
 705 {
 706         struct icl_conn *ic;
 707         size_t available, read = 0;
 708         struct socket *so;
 709         struct mbuf *m, *r = NULL;
 710         struct uio uio;
 711         int error, flags;
 712
 713         ic = arg;
 714         so = ic->ic_socket;
 715
 716         for (;;) {
 717                 SOCKBUF_LOCK(&so->so_rcv);
 718                 if (ic->ic_disconnecting) {
 719                         SOCKBUF_UNLOCK(&so->so_rcv);
 720                         break;
 721                 }
 722
 723                 /*
 724                  * Set the low watermark, to be checked by
 725                  * soreadable() in icl_soupcall_receive()
 726                  * to avoid unnecessary wakeups until there
 727                  * is enough data received to read the PDU.
 728                  */
 729                 available = sbavail(&so->so_rcv);
 730                 if (read + available < ic->ic_receive_len) {
 731                         so->so_rcv.sb_lowat = ic->ic_receive_len - read;
 732                         cv_wait(&ic->ic_receive_cv, &so->so_rcv.sb_mtx);
 733                         so->so_rcv.sb_lowat = so->so_rcv.sb_hiwat + 1;
 734                         available = sbavail(&so->so_rcv);
 735                 }
 736                 SOCKBUF_UNLOCK(&so->so_rcv);
 737
 738                 if (available == 0) {
 739                         if (so->so_error != 0) {
 740                                 ICL_DEBUG("connection error %d; "
 741                                     "dropping connection", so->so_error);
 742                                 icl_conn_fail(ic);
 743                                 break;
 744                         }
 745                         continue;
 746                 }
 747
 748                 memset(&uio, 0, sizeof(uio));
 749                 uio.uio_resid = available;
 750                 flags = MSG_DONTWAIT;
 751                 error = soreceive(so, NULL, &uio, &m, NULL, &flags);
 752                 if (error != 0) {
 753                         ICL_DEBUG("soreceive error %d", error);
 754                         break;
 755                 }
 756                 if (uio.uio_resid != 0) {
 757                         m_freem(m);
 758                         ICL_DEBUG("short read");
 759                         break;
 760                 }
 761                 if (r)
 762                         m_cat(r, m);
 763                 else
 764                         r = m;
 765                 read += available;
 766
 767                 icl_conn_receive_pdus(ic, &r, &read);
 768         }
 769
 770         if (r)
 771                 m_freem(r);
 772
 773         ICL_CONN_LOCK(ic);
 774         ic->ic_receive_running = false;
 775         cv_signal(&ic->ic_send_cv);
 776         ICL_CONN_UNLOCK(ic);
 777         kthread_exit();
 778 }
 779
 780 static int
 781 icl_soupcall_receive(struct socket *so, void *arg, int waitflag)
 782 {
 783         struct icl_conn *ic;
 784
 785         if (!soreadable(so))
 786                 return (SU_OK);
 787
 788         ic = arg;
 789         cv_signal(&ic->ic_receive_cv);
 790         return (SU_OK);
 791 }
 792
 793 static int
 794 icl_pdu_finalize(struct icl_pdu *request)
 795 {
 796         size_t padding, pdu_len;
 797         uint32_t digest, zero = 0;
 798         int ok;
 799         struct icl_conn *ic;
 800
 801         ic = request->ip_conn;
 802
 803         icl_pdu_set_data_segment_length(request, request->ip_data_len);
 804
 805         pdu_len = icl_pdu_size(request);
 806
 807         if (ic->ic_header_crc32c) {
 808                 digest = icl_mbuf_to_crc32c(request->ip_bhs_mbuf);
 809                 ok = m_append(request->ip_bhs_mbuf, sizeof(digest),
 810                     (void *)&digest);
 811                 if (ok != 1) {
 812                         ICL_WARN("failed to append header digest");
 813                         return (1);
 814                 }
 815         }
 816
 817         if (request->ip_data_len != 0) {
 818                 padding = icl_pdu_padding(request);
 819                 if (padding > 0) {
 820                         ok = m_append(request->ip_data_mbuf, padding,
 821                             (void *)&zero);
 822                         if (ok != 1) {
 823                                 ICL_WARN("failed to append padding");
 824                                 return (1);
 825                         }
 826                 }
 827
 828                 if (ic->ic_data_crc32c) {
 829                         digest = icl_mbuf_to_crc32c(request->ip_data_mbuf);
 830
 831                         ok = m_append(request->ip_data_mbuf, sizeof(digest),
 832                             (void *)&digest);
 833                         if (ok != 1) {
 834                                 ICL_WARN("failed to append data digest");
 835                                 return (1);
 836                         }
 837                 }
 838
 839                 m_cat(request->ip_bhs_mbuf, request->ip_data_mbuf);
 840                 request->ip_data_mbuf = NULL;
 841         }
 842
 843         request->ip_bhs_mbuf->m_pkthdr.len = pdu_len;
 844
 845         return (0);
 846 }
 847
 848 static void
 849 icl_conn_send_pdus(struct icl_conn *ic, struct icl_pdu_stailq *queue)
 850 {
 851         struct icl_pdu *request, *request2;
 852         struct mbuf *m;
 853         struct socket *so;
 854         long available, size, size2;
 855         int coalesced, error;
 856
 857         ICL_CONN_LOCK_ASSERT_NOT(ic);
 858
 859         so = ic->ic_socket;
 860
 861         SOCKBUF_LOCK(&so->so_snd);
 862         /*
 863          * Check how much space do we have for transmit.  We can't just
 864          * call sosend() and retry when we get EWOULDBLOCK or EMSGSIZE,
 865          * as it always frees the mbuf chain passed to it, even in case
 866          * of error.
 867          */
 868         available = sbspace(&so->so_snd);
 869         ic->ic_check_send_space = false;
 870
 871         /*
 872          * Notify the socket upcall that we don't need wakeups
 873          * for the time being.
 874          */
 875         so->so_snd.sb_lowat = so->so_snd.sb_hiwat + 1;
 876         SOCKBUF_UNLOCK(&so->so_snd);
 877
 878         while (!STAILQ_EMPTY(queue)) {
 879                 request = STAILQ_FIRST(queue);
 880                 size = icl_pdu_size(request);
 881                 if (available < size) {
 882                         /*
 883                          * Set the low watermark, to be checked by
 884                          * sowriteable() in icl_soupcall_send()
 885                          * to avoid unnecessary wakeups until there
 886                          * is enough space for the PDU to fit.
 887                          */
 888                         SOCKBUF_LOCK(&so->so_snd);
 889                         available = sbspace(&so->so_snd);
 890                         if (available < size) {
 891 #if 1
 892                                 ICL_DEBUG("no space to send; "
 893                                     "have %ld, need %ld",
 894                                     available, size);
 895 #endif
 896                                 so->so_snd.sb_lowat = max(size,
 897                                     so->so_snd.sb_hiwat / 8);
 898                                 SOCKBUF_UNLOCK(&so->so_snd);
 899                                 return;
 900                         }
 901                         SOCKBUF_UNLOCK(&so->so_snd);
 902                 }
 903                 STAILQ_REMOVE_HEAD(queue, ip_next);
 904                 error = icl_pdu_finalize(request);
 905                 if (error != 0) {
 906                         ICL_DEBUG("failed to finalize PDU; "
 907                             "dropping connection");
 908                         icl_soft_pdu_done(request, EIO);
 909                         icl_conn_fail(ic);
 910                         return;
 911                 }
 912                 if (coalesce) {
 913                         m = request->ip_bhs_mbuf;
 914                         for (coalesced = 1; ; coalesced++) {
 915                                 request2 = STAILQ_FIRST(queue);
 916                                 if (request2 == NULL)
 917                                         break;
 918                                 size2 = icl_pdu_size(request2);
 919                                 if (available < size + size2)
 920                                         break;
 921                                 STAILQ_REMOVE_HEAD(queue, ip_next);
 922                                 error = icl_pdu_finalize(request2);
 923                                 if (error != 0) {
 924                                         ICL_DEBUG("failed to finalize PDU; "
 925                                             "dropping connection");
 926                                         icl_soft_pdu_done(request, EIO);
 927                                         icl_soft_pdu_done(request2, EIO);
 928                                         icl_conn_fail(ic);
 929                                         return;
 930                                 }
 931                                 while (m->m_next)
 932                                         m = m->m_next;
 933                                 m_cat(m, request2->ip_bhs_mbuf);
 934                                 request2->ip_bhs_mbuf = NULL;
 935                                 request->ip_bhs_mbuf->m_pkthdr.len += size2;
 936                                 size += size2;
 937                                 icl_soft_pdu_done(request2, 0);
 938                         }
 939 #if 0
 940                         if (coalesced > 1) {
 941                                 ICL_DEBUG("coalesced %d PDUs into %ld bytes",
 942                                     coalesced, size);
 943                         }
 944 #endif
 945                 }
 946                 available -= size;
 947                 error = sosend(so, NULL, NULL, request->ip_bhs_mbuf,
 948                     NULL, MSG_DONTWAIT, curthread);
 949                 request->ip_bhs_mbuf = NULL; /* Sosend consumes the mbuf. */
 950                 if (error != 0) {
 951                         ICL_DEBUG("failed to send PDU, error %d; "
 952                             "dropping connection", error);
 953                         icl_soft_pdu_done(request, error);
 954                         icl_conn_fail(ic);
 955                         return;
 956                 }
 957                 icl_soft_pdu_done(request, 0);
 958         }
 959 }
 960
 961 static void
 962 icl_send_thread(void *arg)
 963 {
 964         struct icl_conn *ic;
 965         struct icl_pdu_stailq queue;
 966
 967         ic = arg;
 968
 969         STAILQ_INIT(&queue);
 970
 971         ICL_CONN_LOCK(ic);
 972         for (;;) {
 973                 for (;;) {
 974                         /*
 975                          * Populate the local queue from the main one.
 976                          * This way the icl_conn_send_pdus() can go through
 977                          * all the queued PDUs without holding any locks.
 978                          */
 979                         if (STAILQ_EMPTY(&queue) || ic->ic_check_send_space)
 980                                 STAILQ_CONCAT(&queue, &ic->ic_to_send);
 981
 982                         ICL_CONN_UNLOCK(ic);
 983                         icl_conn_send_pdus(ic, &queue);
 984                         ICL_CONN_LOCK(ic);
 985
 986                         /*
 987                          * The icl_soupcall_send() was called since the last
 988                          * call to sbspace(); go around;
 989                          */
 990                         if (ic->ic_check_send_space)
 991                                 continue;
 992
 993                         /*
 994                          * Local queue is empty, but we still have PDUs
 995                          * in the main one; go around.
 996                          */
 997                         if (STAILQ_EMPTY(&queue) &&
 998                             !STAILQ_EMPTY(&ic->ic_to_send))
 999                                 continue;
1000
1001                         /*
1002                          * There might be some stuff in the local queue,
1003                          * which didn't get sent due to not having enough send
1004                          * space.  Wait for socket upcall.
1005                          */
1006                         break;
1007                 }
1008
1009                 if (ic->ic_disconnecting) {
1010                         //ICL_DEBUG("terminating");
1011                         break;
1012                 }
1013
1014                 cv_wait(&ic->ic_send_cv, ic->ic_lock);
1015         }
1016
1017         /*
1018          * We're exiting; move PDUs back to the main queue, so they can
1019          * get freed properly.  At this point ordering doesn't matter.
1020          */
1021         STAILQ_CONCAT(&ic->ic_to_send, &queue);
1022
1023         ic->ic_send_running = false;
1024         cv_signal(&ic->ic_send_cv);
1025         ICL_CONN_UNLOCK(ic);
1026         kthread_exit();
1027 }
1028
1029 static int
1030 icl_soupcall_send(struct socket *so, void *arg, int waitflag)
1031 {
1032         struct icl_conn *ic;
1033
1034         if (!sowriteable(so))
1035                 return (SU_OK);
1036
1037         ic = arg;
1038
1039         ICL_CONN_LOCK(ic);
1040         ic->ic_check_send_space = true;
1041         ICL_CONN_UNLOCK(ic);
1042
1043         cv_signal(&ic->ic_send_cv);
1044
1045         return (SU_OK);
1046 }
1047
1048 static int
1049 icl_soft_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request,
1050     const void *addr, size_t len, int flags)
1051 {
1052         struct icl_soft_pdu *isp = (struct icl_soft_pdu *)request;
1053         struct mbuf *mb, *newmb;
1054         size_t copylen, off = 0;
1055
1056         KASSERT(len > 0, ("len == 0"));
1057
1058         if (flags & ICL_NOCOPY) {
1059                 newmb = m_get(flags & ~ICL_NOCOPY, MT_DATA);
1060                 if (newmb == NULL) {
1061                         ICL_WARN("failed to allocate mbuf");
1062                         return (ENOMEM);
1063                 }
1064
1065                 newmb->m_flags |= M_RDONLY;
1066                 m_extaddref(newmb, __DECONST(char *, addr), len, &isp->ref_cnt,
1067                     icl_soft_mbuf_done, isp, NULL);
1068                 newmb->m_len = len;
1069         } else {
1070                 newmb = m_getm2(NULL, len, flags, MT_DATA, 0);
1071                 if (newmb == NULL) {
1072                         ICL_WARN("failed to allocate mbuf for %zd bytes", len);
1073                         return (ENOMEM);
1074                 }
1075
1076                 for (mb = newmb; mb != NULL; mb = mb->m_next) {
1077                         copylen = min(M_TRAILINGSPACE(mb), len - off);
1078                         memcpy(mtod(mb, char *), (const char *)addr + off, copylen);
1079                         mb->m_len = copylen;
1080                         off += copylen;
1081                 }
1082                 KASSERT(off == len, ("%s: off != len", __func__));
1083         }
1084
1085         if (request->ip_data_mbuf == NULL) {
1086                 request->ip_data_mbuf = newmb;
1087                 request->ip_data_len = len;
1088         } else {
1089                 m_cat(request->ip_data_mbuf, newmb);
1090                 request->ip_data_len += len;
1091         }
1092
1093         return (0);
1094 }
1095
1096 void
1097 icl_soft_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
1098     size_t off, void *addr, size_t len)
1099 {
1100
1101         m_copydata(ip->ip_data_mbuf, off, len, addr);
1102 }
1103
1104 static void
1105 icl_soft_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip)
1106 {
1107
1108         icl_soft_conn_pdu_queue_cb(ic, ip, NULL);
1109 }
1110
1111 static void
1112 icl_soft_conn_pdu_queue_cb(struct icl_conn *ic, struct icl_pdu *ip,
1113     icl_pdu_cb cb)
1114 {
1115         struct icl_soft_pdu *isp = (struct icl_soft_pdu *)ip;
1116
1117         ICL_CONN_LOCK_ASSERT(ic);
1118         isp->ref_cnt++;
1119         isp->cb = cb;
1120
1121         if (ic->ic_disconnecting || ic->ic_socket == NULL) {
1122                 ICL_DEBUG("icl_pdu_queue on closed connection");
1123                 icl_soft_pdu_done(ip, ENOTCONN);
1124                 return;
1125         }
1126
1127         if (!STAILQ_EMPTY(&ic->ic_to_send)) {
1128                 STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next);
1129                 /*
1130                  * If the queue is not empty, someone else had already
1131                  * signaled the send thread; no need to do that again,
1132                  * just return.
1133                  */
1134                 return;
1135         }
1136
1137         STAILQ_INSERT_TAIL(&ic->ic_to_send, ip, ip_next);
1138         cv_signal(&ic->ic_send_cv);
1139 }
1140
1141 static struct icl_conn *
1142 icl_soft_new_conn(const char *name, struct mtx *lock)
1143 {
1144         struct icl_conn *ic;
1145
1146         refcount_acquire(&icl_ncons);
1147
1148         ic = (struct icl_conn *)kobj_create(&icl_soft_class, M_ICL_SOFT, M_WAITOK | M_ZERO);
1149
1150         STAILQ_INIT(&ic->ic_to_send);
1151         ic->ic_lock = lock;
1152         cv_init(&ic->ic_send_cv, "icl_tx");
1153         cv_init(&ic->ic_receive_cv, "icl_rx");
1154 #ifdef DIAGNOSTIC
1155         refcount_init(&ic->ic_outstanding_pdus, 0);
1156 #endif
1157         ic->ic_name = name;
1158         ic->ic_offload = "None";
1159         ic->ic_unmapped = false;
1160
1161         return (ic);
1162 }
1163
1164 void
1165 icl_soft_conn_free(struct icl_conn *ic)
1166 {
1167
1168 #ifdef DIAGNOSTIC
1169         KASSERT(ic->ic_outstanding_pdus == 0,
1170             ("destroying session with %d outstanding PDUs",
1171              ic->ic_outstanding_pdus));
1172 #endif
1173         cv_destroy(&ic->ic_send_cv);
1174         cv_destroy(&ic->ic_receive_cv);
1175         kobj_delete((struct kobj *)ic, M_ICL_SOFT);
1176         refcount_release(&icl_ncons);
1177 }
1178
1179 static int
1180 icl_conn_start(struct icl_conn *ic)
1181 {
1182         size_t minspace;
1183         struct sockopt opt;
1184         int error, one = 1;
1185
1186         ICL_CONN_LOCK(ic);
1187
1188         /*
1189          * XXX: Ugly hack.
1190          */
1191         if (ic->ic_socket == NULL) {
1192                 ICL_CONN_UNLOCK(ic);
1193                 return (EINVAL);
1194         }
1195
1196         ic->ic_receive_state = ICL_CONN_STATE_BHS;
1197         ic->ic_receive_len = sizeof(struct iscsi_bhs);
1198         ic->ic_disconnecting = false;
1199
1200         ICL_CONN_UNLOCK(ic);
1201
1202         /*
1203          * For sendspace, this is required because the current code cannot
1204          * send a PDU in pieces; thus, the minimum buffer size is equal
1205          * to the maximum PDU size.  "+4" is to account for possible padding.
1206          */
1207         minspace = sizeof(struct iscsi_bhs) +
1208             ic->ic_max_send_data_segment_length +
1209             ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4;
1210         if (sendspace < minspace) {
1211                 ICL_WARN("kern.icl.sendspace too low; must be at least %zd",
1212                     minspace);
1213                 sendspace = minspace;
1214         }
1215         minspace = sizeof(struct iscsi_bhs) +
1216             ic->ic_max_recv_data_segment_length +
1217             ISCSI_HEADER_DIGEST_SIZE + ISCSI_DATA_DIGEST_SIZE + 4;
1218         if (recvspace < minspace) {
1219                 ICL_WARN("kern.icl.recvspace too low; must be at least %zd",
1220                     minspace);
1221                 recvspace = minspace;
1222         }
1223
1224         error = soreserve(ic->ic_socket, sendspace, recvspace);
1225         if (error != 0) {
1226                 ICL_WARN("soreserve failed with error %d", error);
1227                 icl_soft_conn_close(ic);
1228                 return (error);
1229         }
1230         ic->ic_socket->so_snd.sb_flags |= SB_AUTOSIZE;
1231         ic->ic_socket->so_rcv.sb_flags |= SB_AUTOSIZE;
1232
1233         /*
1234          * Disable Nagle.
1235          */
1236         bzero(&opt, sizeof(opt));
1237         opt.sopt_dir = SOPT_SET;
1238         opt.sopt_level = IPPROTO_TCP;
1239         opt.sopt_name = TCP_NODELAY;
1240         opt.sopt_val = &one;
1241         opt.sopt_valsize = sizeof(one);
1242         error = sosetopt(ic->ic_socket, &opt);
1243         if (error != 0) {
1244                 ICL_WARN("disabling TCP_NODELAY failed with error %d", error);
1245                 icl_soft_conn_close(ic);
1246                 return (error);
1247         }
1248
1249         /*
1250          * Register socket upcall, to get notified about incoming PDUs
1251          * and free space to send outgoing ones.
1252          */
1253         SOCKBUF_LOCK(&ic->ic_socket->so_snd);
1254         soupcall_set(ic->ic_socket, SO_SND, icl_soupcall_send, ic);
1255         SOCKBUF_UNLOCK(&ic->ic_socket->so_snd);
1256         SOCKBUF_LOCK(&ic->ic_socket->so_rcv);
1257         soupcall_set(ic->ic_socket, SO_RCV, icl_soupcall_receive, ic);
1258         SOCKBUF_UNLOCK(&ic->ic_socket->so_rcv);
1259
1260         /*
1261          * Start threads.
1262          */
1263         ICL_CONN_LOCK(ic);
1264         ic->ic_send_running = ic->ic_receive_running = true;
1265         ICL_CONN_UNLOCK(ic);
1266         error = kthread_add(icl_send_thread, ic, NULL, NULL, 0, 0, "%stx",
1267             ic->ic_name);
1268         if (error != 0) {
1269                 ICL_WARN("kthread_add(9) failed with error %d", error);
1270                 ICL_CONN_LOCK(ic);
1271                 ic->ic_send_running = ic->ic_receive_running = false;
1272                 cv_signal(&ic->ic_send_cv);
1273                 ICL_CONN_UNLOCK(ic);
1274                 icl_soft_conn_close(ic);
1275                 return (error);
1276         }
1277         error = kthread_add(icl_receive_thread, ic, NULL, NULL, 0, 0, "%srx",
1278             ic->ic_name);
1279         if (error != 0) {
1280                 ICL_WARN("kthread_add(9) failed with error %d", error);
1281                 ICL_CONN_LOCK(ic);
1282                 ic->ic_receive_running = false;
1283                 cv_signal(&ic->ic_send_cv);
1284                 ICL_CONN_UNLOCK(ic);
1285                 icl_soft_conn_close(ic);
1286                 return (error);
1287         }
1288
1289         return (0);
1290 }
1291
1292 int
1293 icl_soft_conn_handoff(struct icl_conn *ic, int fd)
1294 {
1295         struct file *fp;
1296         struct socket *so;
1297         cap_rights_t rights;
1298         int error;
1299
1300         ICL_CONN_LOCK_ASSERT_NOT(ic);
1301
1302 #ifdef ICL_KERNEL_PROXY
1303         /*
1304          * We're transitioning to Full Feature phase, and we don't
1305          * really care.
1306          */
1307         if (fd == 0) {
1308                 ICL_CONN_LOCK(ic);
1309                 if (ic->ic_socket == NULL) {
1310                         ICL_CONN_UNLOCK(ic);
1311                         ICL_WARN("proxy handoff without connect");
1312                         return (EINVAL);
1313                 }
1314                 ICL_CONN_UNLOCK(ic);
1315                 return (0);
1316         }
1317 #endif
1318
1319         /*
1320          * Steal the socket from userland.
1321          */
1322         error = fget(curthread, fd,
1323             cap_rights_init_one(&rights, CAP_SOCK_CLIENT), &fp);
1324         if (error != 0)
1325                 return (error);
1326         if (fp->f_type != DTYPE_SOCKET) {
1327                 fdrop(fp, curthread);
1328                 return (EINVAL);
1329         }
1330         so = fp->f_data;
1331         if (so->so_type != SOCK_STREAM) {
1332                 fdrop(fp, curthread);
1333                 return (EINVAL);
1334         }
1335
1336         ICL_CONN_LOCK(ic);
1337
1338         if (ic->ic_socket != NULL) {
1339                 ICL_CONN_UNLOCK(ic);
1340                 fdrop(fp, curthread);
1341                 return (EBUSY);
1342         }
1343
1344         ic->ic_socket = fp->f_data;
1345         fp->f_ops = &badfileops;
1346         fp->f_data = NULL;
1347         fdrop(fp, curthread);
1348         ICL_CONN_UNLOCK(ic);
1349
1350         error = icl_conn_start(ic);
1351
1352         return (error);
1353 }
1354
1355 void
1356 icl_soft_conn_close(struct icl_conn *ic)
1357 {
1358         struct icl_pdu *pdu;
1359         struct socket *so;
1360
1361         /*
1362          * Wake up the threads, so they can properly terminate.
1363          * Receive thread sleeps on so->so_rcv lock, send on ic->ic_lock.
1364          */
1365         ICL_CONN_LOCK(ic);
1366         if (!ic->ic_disconnecting) {
1367                 so = ic->ic_socket;
1368                 if (so)
1369                         SOCKBUF_LOCK(&so->so_rcv);
1370                 ic->ic_disconnecting = true;
1371                 if (so)
1372                         SOCKBUF_UNLOCK(&so->so_rcv);
1373         }
1374         while (ic->ic_receive_running || ic->ic_send_running) {
1375                 cv_signal(&ic->ic_receive_cv);
1376                 cv_signal(&ic->ic_send_cv);
1377                 cv_wait(&ic->ic_send_cv, ic->ic_lock);
1378         }
1379
1380         /* Some other thread could close the connection same time. */
1381         so = ic->ic_socket;
1382         if (so == NULL) {
1383                 ICL_CONN_UNLOCK(ic);
1384                 return;
1385         }
1386         ic->ic_socket = NULL;
1387
1388         /*
1389          * Deregister socket upcalls.
1390          */
1391         ICL_CONN_UNLOCK(ic);
1392         SOCKBUF_LOCK(&so->so_snd);
1393         if (so->so_snd.sb_upcall != NULL)
1394                 soupcall_clear(so, SO_SND);
1395         SOCKBUF_UNLOCK(&so->so_snd);
1396         SOCKBUF_LOCK(&so->so_rcv);
1397         if (so->so_rcv.sb_upcall != NULL)
1398                 soupcall_clear(so, SO_RCV);
1399         SOCKBUF_UNLOCK(&so->so_rcv);
1400         soclose(so);
1401         ICL_CONN_LOCK(ic);
1402
1403         if (ic->ic_receive_pdu != NULL) {
1404                 //ICL_DEBUG("freeing partially received PDU");
1405                 icl_soft_conn_pdu_free(ic, ic->ic_receive_pdu);
1406                 ic->ic_receive_pdu = NULL;
1407         }
1408
1409         /*
1410          * Remove any outstanding PDUs from the send queue.
1411          */
1412         while (!STAILQ_EMPTY(&ic->ic_to_send)) {
1413                 pdu = STAILQ_FIRST(&ic->ic_to_send);
1414                 STAILQ_REMOVE_HEAD(&ic->ic_to_send, ip_next);
1415                 icl_soft_pdu_done(pdu, ENOTCONN);
1416         }
1417
1418         KASSERT(STAILQ_EMPTY(&ic->ic_to_send),
1419             ("destroying session with non-empty send queue"));
1420         ICL_CONN_UNLOCK(ic);
1421 }
1422
1423 int
1424 icl_soft_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
1425     struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp)
1426 {
1427
1428         return (0);
1429 }
1430
1431 void
1432 icl_soft_conn_task_done(struct icl_conn *ic, void *prv)
1433 {
1434 }
1435
1436 int
1437 icl_soft_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
1438     uint32_t *transfer_tag, void **prvp)
1439 {
1440
1441         return (0);
1442 }
1443
1444 void
1445 icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv)
1446 {
1447 }
1448
1449 static int
1450 icl_soft_limits(struct icl_drv_limits *idl)
1451 {
1452
1453         idl->idl_max_recv_data_segment_length = max_data_segment_length;
1454         idl->idl_max_send_data_segment_length = max_data_segment_length;
1455         idl->idl_max_burst_length = max_burst_length;
1456         idl->idl_first_burst_length = first_burst_length;
1457
1458         return (0);
1459 }
1460
1461 #ifdef ICL_KERNEL_PROXY
1462 int
1463 icl_soft_conn_connect(struct icl_conn *ic, int domain, int socktype,
1464     int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa)
1465 {
1466
1467         return (icl_soft_proxy_connect(ic, domain, socktype, protocol,
1468             from_sa, to_sa));
1469 }
1470
1471 int
1472 icl_soft_handoff_sock(struct icl_conn *ic, struct socket *so)
1473 {
1474         int error;
1475
1476         ICL_CONN_LOCK_ASSERT_NOT(ic);
1477
1478         if (so->so_type != SOCK_STREAM)
1479                 return (EINVAL);
1480
1481         ICL_CONN_LOCK(ic);
1482         if (ic->ic_socket != NULL) {
1483                 ICL_CONN_UNLOCK(ic);
1484                 return (EBUSY);
1485         }
1486         ic->ic_socket = so;
1487         ICL_CONN_UNLOCK(ic);
1488
1489         error = icl_conn_start(ic);
1490
1491         return (error);
1492 }
1493 #endif /* ICL_KERNEL_PROXY */
1494
1495 static int
1496 icl_soft_load(void)
1497 {
1498         int error;
1499
1500         icl_soft_pdu_zone = uma_zcreate("icl_soft_pdu",
1501             sizeof(struct icl_soft_pdu), NULL, NULL, NULL, NULL,
1502             UMA_ALIGN_PTR, 0);
1503         refcount_init(&icl_ncons, 0);
1504
1505         /*
1506          * The reason we call this "none" is that to the user,
1507          * it's known as "offload driver"; "offload driver: soft"
1508          * doesn't make much sense.
1509          */
1510         error = icl_register("none", false, 0,
1511             icl_soft_limits, icl_soft_new_conn);
1512         KASSERT(error == 0, ("failed to register"));
1513
1514 #if defined(ICL_KERNEL_PROXY) && 0
1515         /*
1516          * Debugging aid for kernel proxy functionality.
1517          */
1518         error = icl_register("proxytest", true, 0,
1519             icl_soft_limits, icl_soft_new_conn);
1520         KASSERT(error == 0, ("failed to register"));
1521 #endif
1522
1523         return (error);
1524 }
1525
1526 static int
1527 icl_soft_unload(void)
1528 {
1529
1530         if (icl_ncons != 0)
1531                 return (EBUSY);
1532
1533         icl_unregister("none", false);
1534 #if defined(ICL_KERNEL_PROXY) && 0
1535         icl_unregister("proxytest", true);
1536 #endif
1537
1538         uma_zdestroy(icl_soft_pdu_zone);
1539
1540         return (0);
1541 }
1542
1543 static int
1544 icl_soft_modevent(module_t mod, int what, void *arg)
1545 {
1546
1547         switch (what) {
1548         case MOD_LOAD:
1549                 return (icl_soft_load());
1550         case MOD_UNLOAD:
1551                 return (icl_soft_unload());
1552         default:
1553                 return (EINVAL);
1554         }
1555 }
1556
1557 moduledata_t icl_soft_data = {
1558         "icl_soft",
1559         icl_soft_modevent,
1560         0
1561 };
1562
1563 DECLARE_MODULE(icl_soft, icl_soft_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
1564 MODULE_DEPEND(icl_soft, icl, 1, 1, 1);
1565 MODULE_VERSION(icl_soft, 1);