2 * Copyright (c) 2012 The FreeBSD Foundation
3 * Copyright (c) 2015 Chelsio Communications, Inc.
6 * This software was developed by Edward Tomasz Napierala under sponsorship
7 * from the FreeBSD Foundation.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 * cxgbei implementation of iSCSI Common Layer kobj(9) interface.
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
40 #include "opt_inet6.h"
43 #include <sys/param.h>
44 #include <sys/capsicum.h>
45 #include <sys/condvar.h>
48 #include <sys/kernel.h>
49 #include <sys/kthread.h>
52 #include <sys/mutex.h>
53 #include <sys/module.h>
54 #include <sys/protosw.h>
55 #include <sys/socket.h>
56 #include <sys/socketvar.h>
57 #include <sys/sysctl.h>
58 #include <sys/systm.h>
61 #include <machine/bus.h>
65 #include <netinet/in.h>
66 #include <netinet/in_pcb.h>
67 #include <netinet/tcp.h>
68 #include <netinet/tcp_var.h>
69 #include <netinet/toecore.h>
71 #include <dev/iscsi/icl.h>
72 #include <dev/iscsi/iscsi_proto.h>
73 #include <icl_conn_if.h>
75 #include <cam/scsi/scsi_all.h>
76 #include <cam/scsi/scsi_da.h>
77 #include <cam/ctl/ctl_io.h>
78 #include <cam/ctl/ctl.h>
79 #include <cam/ctl/ctl_backend.h>
80 #include <cam/ctl/ctl_error.h>
81 #include <cam/ctl/ctl_frontend.h>
82 #include <cam/ctl/ctl_debug.h>
83 #include <cam/ctl/ctl_ha.h>
84 #include <cam/ctl/ctl_ioctl.h>
87 #include <cam/cam_ccb.h>
88 #include <cam/cam_xpt.h>
89 #include <cam/cam_debug.h>
90 #include <cam/cam_sim.h>
91 #include <cam/cam_xpt_sim.h>
92 #include <cam/cam_xpt_periph.h>
93 #include <cam/cam_periph.h>
94 #include <cam/cam_compat.h>
95 #include <cam/scsi/scsi_message.h>
97 #include "common/common.h"
98 #include "common/t4_tcb.h"
99 #include "tom/t4_tom.h"
102 SYSCTL_NODE(_kern_icl, OID_AUTO, cxgbei, CTLFLAG_RD, 0, "Chelsio iSCSI offload");
103 static int coalesce = 1;
104 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, coalesce, CTLFLAG_RWTUN,
105 &coalesce, 0, "Try to coalesce PDUs before sending");
106 static int partial_receive_len = 128 * 1024;
107 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, partial_receive_len, CTLFLAG_RWTUN,
108 &partial_receive_len, 0, "Minimum read size for partially received "
110 static int sendspace = 1048576;
111 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, sendspace, CTLFLAG_RWTUN,
112 &sendspace, 0, "Default send socket buffer size");
113 static int recvspace = 1048576;
114 SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN,
115 &recvspace, 0, "Default receive socket buffer size");
117 static uma_zone_t prsv_zone;
118 static volatile u_int icl_cxgbei_ncons;
120 #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock)
121 #define ICL_CONN_UNLOCK(X) mtx_unlock(X->ic_lock)
122 #define ICL_CONN_LOCK_ASSERT(X) mtx_assert(X->ic_lock, MA_OWNED)
123 #define ICL_CONN_LOCK_ASSERT_NOT(X) mtx_assert(X->ic_lock, MA_NOTOWNED)
125 struct icl_pdu *icl_cxgbei_new_pdu(int);
126 void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *);
128 static icl_conn_new_pdu_t icl_cxgbei_conn_new_pdu;
129 icl_conn_pdu_free_t icl_cxgbei_conn_pdu_free;
130 static icl_conn_pdu_data_segment_length_t
131 icl_cxgbei_conn_pdu_data_segment_length;
132 static icl_conn_pdu_append_data_t icl_cxgbei_conn_pdu_append_data;
133 static icl_conn_pdu_get_data_t icl_cxgbei_conn_pdu_get_data;
134 static icl_conn_pdu_queue_t icl_cxgbei_conn_pdu_queue;
135 static icl_conn_handoff_t icl_cxgbei_conn_handoff;
136 static icl_conn_free_t icl_cxgbei_conn_free;
137 static icl_conn_close_t icl_cxgbei_conn_close;
138 static icl_conn_task_setup_t icl_cxgbei_conn_task_setup;
139 static icl_conn_task_done_t icl_cxgbei_conn_task_done;
140 static icl_conn_transfer_setup_t icl_cxgbei_conn_transfer_setup;
141 static icl_conn_transfer_done_t icl_cxgbei_conn_transfer_done;
143 static kobj_method_t icl_cxgbei_methods[] = {
144 KOBJMETHOD(icl_conn_new_pdu, icl_cxgbei_conn_new_pdu),
145 KOBJMETHOD(icl_conn_pdu_free, icl_cxgbei_conn_pdu_free),
146 KOBJMETHOD(icl_conn_pdu_data_segment_length,
147 icl_cxgbei_conn_pdu_data_segment_length),
148 KOBJMETHOD(icl_conn_pdu_append_data, icl_cxgbei_conn_pdu_append_data),
149 KOBJMETHOD(icl_conn_pdu_get_data, icl_cxgbei_conn_pdu_get_data),
150 KOBJMETHOD(icl_conn_pdu_queue, icl_cxgbei_conn_pdu_queue),
151 KOBJMETHOD(icl_conn_handoff, icl_cxgbei_conn_handoff),
152 KOBJMETHOD(icl_conn_free, icl_cxgbei_conn_free),
153 KOBJMETHOD(icl_conn_close, icl_cxgbei_conn_close),
154 KOBJMETHOD(icl_conn_task_setup, icl_cxgbei_conn_task_setup),
155 KOBJMETHOD(icl_conn_task_done, icl_cxgbei_conn_task_done),
156 KOBJMETHOD(icl_conn_transfer_setup, icl_cxgbei_conn_transfer_setup),
157 KOBJMETHOD(icl_conn_transfer_done, icl_cxgbei_conn_transfer_done),
161 DEFINE_CLASS(icl_cxgbei, icl_cxgbei_methods, sizeof(struct icl_cxgbei_conn));
164 icl_cxgbei_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
167 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
170 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
171 MPASS(ic == ip->ip_conn);
172 MPASS(ip->ip_bhs_mbuf != NULL);
174 m_freem(ip->ip_ahs_mbuf);
175 m_freem(ip->ip_data_mbuf);
176 m_freem(ip->ip_bhs_mbuf); /* storage for icl_cxgbei_pdu itself */
179 if (__predict_true(ic != NULL))
180 refcount_release(&ic->ic_outstanding_pdus);
185 icl_cxgbei_new_pdu(int flags)
187 struct icl_cxgbei_pdu *icp;
192 m = m_gethdr(flags, MT_DATA);
193 if (__predict_false(m == NULL))
196 a = roundup2(mtod(m, uintptr_t), _Alignof(struct icl_cxgbei_pdu));
197 icp = (struct icl_cxgbei_pdu *)a;
198 bzero(icp, sizeof(*icp));
200 icp->icp_signature = CXGBEI_PDU_SIGNATURE;
204 a = roundup2((uintptr_t)(icp + 1), _Alignof(struct iscsi_bhs *));
205 ip->ip_bhs = (struct iscsi_bhs *)a;
207 /* Everything must fit entirely in the mbuf. */
208 a = (uintptr_t)(ip->ip_bhs + 1);
209 MPASS(a <= (uintptr_t)m + MSIZE);
211 bzero(ip->ip_bhs, sizeof(*ip->ip_bhs));
213 m->m_data = (void *)ip->ip_bhs;
214 m->m_len = sizeof(struct iscsi_bhs);
215 m->m_pkthdr.len = m->m_len;
221 icl_cxgbei_new_pdu_set_conn(struct icl_pdu *ip, struct icl_conn *ic)
226 refcount_acquire(&ic->ic_outstanding_pdus);
231 * Allocate icl_pdu with empty BHS to fill up by the caller.
233 static struct icl_pdu *
234 icl_cxgbei_conn_new_pdu(struct icl_conn *ic, int flags)
238 ip = icl_cxgbei_new_pdu(flags);
239 if (__predict_false(ip == NULL))
241 icl_cxgbei_new_pdu_set_conn(ip, ic);
247 icl_pdu_data_segment_length(const struct icl_pdu *request)
251 len += request->ip_bhs->bhs_data_segment_len[0];
253 len += request->ip_bhs->bhs_data_segment_len[1];
255 len += request->ip_bhs->bhs_data_segment_len[2];
261 icl_cxgbei_conn_pdu_data_segment_length(struct icl_conn *ic,
262 const struct icl_pdu *request)
265 return (icl_pdu_data_segment_length(request));
269 finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp)
271 struct icl_pdu *ip = &icp->ip;
272 uint8_t ulp_submode, padding;
273 struct mbuf *m, *last;
274 struct iscsi_bhs *bhs;
277 * Fix up the data segment mbuf first.
279 m = ip->ip_data_mbuf;
280 ulp_submode = icc->ulp_submode;
285 * Round up the data segment to a 4B boundary. Pad with 0 if
286 * necessary. There will definitely be room in the mbuf.
288 padding = roundup2(ip->ip_data_len, 4) - ip->ip_data_len;
290 bzero(mtod(last, uint8_t *) + last->m_len, padding);
291 last->m_len += padding;
294 MPASS(ip->ip_data_len == 0);
295 ulp_submode &= ~ULP_CRC_DATA;
300 * Now the header mbuf that has the BHS.
303 MPASS(m->m_pkthdr.len == sizeof(struct iscsi_bhs));
304 MPASS(m->m_len == sizeof(struct iscsi_bhs));
307 bhs->bhs_data_segment_len[2] = ip->ip_data_len;
308 bhs->bhs_data_segment_len[1] = ip->ip_data_len >> 8;
309 bhs->bhs_data_segment_len[0] = ip->ip_data_len >> 16;
311 /* "Convert" PDU to mbuf chain. Do not use icp/ip after this. */
312 m->m_pkthdr.len = sizeof(struct iscsi_bhs) + ip->ip_data_len + padding;
313 m->m_next = ip->ip_data_mbuf;
314 set_mbuf_ulp_submode(m, ulp_submode);
316 bzero(icp, sizeof(*icp));
319 refcount_release(&icc->ic.ic_outstanding_pdus);
326 icl_cxgbei_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *ip,
327 const void *addr, size_t len, int flags)
331 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
334 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
335 MPASS(ic == ip->ip_conn);
336 KASSERT(len > 0, ("%s: len is %jd", __func__, (intmax_t)len));
338 m = ip->ip_data_mbuf;
340 m = m_getjcl(M_NOWAIT, MT_DATA, 0, MJUM16BYTES);
341 if (__predict_false(m == NULL))
344 ip->ip_data_mbuf = m;
347 if (__predict_true(m_append(m, len, addr) != 0)) {
348 ip->ip_data_len += len;
349 MPASS(ip->ip_data_len <= ic->ic_max_data_segment_length);
352 if (flags & M_WAITOK) {
353 CXGBE_UNIMPLEMENTED("fail safe append");
355 ip->ip_data_len = m_length(m, NULL);
361 icl_cxgbei_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
362 size_t off, void *addr, size_t len)
364 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
366 if (icp->icp_flags & ICPF_RX_DDP)
367 return; /* data is DDP'ed, no need to copy */
368 m_copydata(ip->ip_data_mbuf, off, len, addr);
372 icl_cxgbei_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip)
374 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
375 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
376 struct socket *so = ic->ic_socket;
377 struct toepcb *toep = icc->toep;
381 MPASS(ic == ip->ip_conn);
382 MPASS(ip->ip_bhs_mbuf != NULL);
383 /* The kernel doesn't generate PDUs with AHS. */
384 MPASS(ip->ip_ahs_mbuf == NULL && ip->ip_ahs_len == 0);
386 ICL_CONN_LOCK_ASSERT(ic);
387 /* NOTE: sowriteable without so_snd lock is a mostly harmless race. */
388 if (ic->ic_disconnecting || so == NULL || !sowriteable(so)) {
389 icl_cxgbei_conn_pdu_free(ic, ip);
393 m = finalize_pdu(icc, icp);
395 MPASS((m->m_pkthdr.len & 3) == 0);
398 * Do not get inp from toep->inp as the toepcb might have detached
403 if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) ||
404 __predict_false((toep->flags & TPF_ATTACHED) == 0))
407 mbufq_enqueue(&toep->ulp_pduq, m);
408 t4_push_pdus(icc->sc, toep, 0);
413 static struct icl_conn *
414 icl_cxgbei_new_conn(const char *name, struct mtx *lock)
416 struct icl_cxgbei_conn *icc;
419 refcount_acquire(&icl_cxgbei_ncons);
421 icc = (struct icl_cxgbei_conn *)kobj_create(&icl_cxgbei_class, M_CXGBE,
423 icc->icc_signature = CXGBEI_CONN_SIGNATURE;
424 STAILQ_INIT(&icc->rcvd_pdus);
429 /* XXXNP: review. Most of these icl_conn fields aren't really used */
430 STAILQ_INIT(&ic->ic_to_send);
431 cv_init(&ic->ic_send_cv, "icl_cxgbei_tx");
432 cv_init(&ic->ic_receive_cv, "icl_cxgbei_rx");
434 refcount_init(&ic->ic_outstanding_pdus, 0);
436 /* This is a stop-gap value that will be corrected during handoff. */
437 ic->ic_max_data_segment_length = 16384;
439 ic->ic_offload = "cxgbei";
440 ic->ic_unmapped = false;
442 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);
448 icl_cxgbei_conn_free(struct icl_conn *ic)
450 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
452 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
454 CTR2(KTR_CXGBE, "%s: icc %p", __func__, icc);
456 cv_destroy(&ic->ic_send_cv);
457 cv_destroy(&ic->ic_receive_cv);
459 kobj_delete((struct kobj *)icc, M_CXGBE);
460 refcount_release(&icl_cxgbei_ncons);
464 icl_cxgbei_setsockopt(struct icl_conn *ic, struct socket *so, int sspace,
468 int error, one = 1, ss, rs;
470 ss = max(sendspace, sspace);
471 rs = max(recvspace, rspace);
473 error = soreserve(so, ss, rs);
475 icl_cxgbei_conn_close(ic);
478 SOCKBUF_LOCK(&so->so_snd);
479 so->so_snd.sb_flags |= SB_AUTOSIZE;
480 SOCKBUF_UNLOCK(&so->so_snd);
481 SOCKBUF_LOCK(&so->so_rcv);
482 so->so_rcv.sb_flags |= SB_AUTOSIZE;
483 SOCKBUF_UNLOCK(&so->so_rcv);
488 bzero(&opt, sizeof(opt));
489 opt.sopt_dir = SOPT_SET;
490 opt.sopt_level = IPPROTO_TCP;
491 opt.sopt_name = TCP_NODELAY;
493 opt.sopt_valsize = sizeof(one);
494 error = sosetopt(so, &opt);
496 icl_cxgbei_conn_close(ic);
504 * Request/response structure used to find out the adapter offloading a socket.
506 struct find_ofld_adapter_rr {
508 struct adapter *sc; /* result */
512 find_offload_adapter(struct adapter *sc, void *arg)
514 struct find_ofld_adapter_rr *fa = arg;
515 struct socket *so = fa->so;
516 struct tom_data *td = sc->tom_softc;
520 /* Non-TCP were filtered out earlier. */
521 MPASS(so->so_proto->pr_protocol == IPPROTO_TCP);
524 return; /* Found already. */
527 return; /* TOE not enabled on this adapter. */
531 if ((inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) == 0) {
533 if (tp->t_flags & TF_TOE && tp->tod == &td->tod)
534 fa->sc = sc; /* Found. */
539 /* XXXNP: move this to t4_tom. */
541 send_iscsi_flowc_wr(struct adapter *sc, struct toepcb *toep, int maxlen)
544 struct fw_flowc_wr *flowc;
545 const u_int nparams = 1;
547 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
549 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
551 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
554 panic("%s: allocation failure.", __func__);
557 memset(flowc, 0, wr->wr_len);
559 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
560 V_FW_FLOWC_WR_NPARAMS(nparams));
561 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
562 V_FW_WR_FLOWID(toep->tid));
564 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
565 flowc->mnemval[0].val = htobe32(maxlen);
567 txsd->tx_credits = howmany(flowclen, 16);
569 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
570 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
571 toep->tx_credits -= txsd->tx_credits;
572 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
580 set_ulp_mode_iscsi(struct adapter *sc, struct toepcb *toep, int hcrc, int dcrc)
582 uint64_t val = ULP_MODE_ISCSI;
585 val |= ULP_CRC_HEADER << 4;
587 val |= ULP_CRC_DATA << 4;
589 CTR4(KTR_CXGBE, "%s: tid %u, ULP_MODE_ISCSI, CRC hdr=%d data=%d",
590 __func__, toep->tid, hcrc, dcrc);
592 t4_set_tcb_field(sc, toep->ctrlq, toep, W_TCB_ULP_TYPE,
593 V_TCB_ULP_TYPE(M_TCB_ULP_TYPE) | V_TCB_ULP_RAW(M_TCB_ULP_RAW), val,
598 * XXXNP: Who is responsible for cleaning up the socket if this returns with an
599 * error? Review all error paths.
601 * XXXNP: What happens to the socket's fd reference if the operation is
602 * successful, and how does that affect the socket's life cycle?
605 icl_cxgbei_conn_handoff(struct icl_conn *ic, int fd)
607 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
608 struct cxgbei_data *ci;
609 struct find_ofld_adapter_rr fa;
618 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
619 ICL_CONN_LOCK_ASSERT_NOT(ic);
622 * Steal the socket from userland.
624 error = fget(curthread, fd,
625 cap_rights_init(&rights, CAP_SOCK_CLIENT), &fp);
628 if (fp->f_type != DTYPE_SOCKET) {
629 fdrop(fp, curthread);
633 if (so->so_type != SOCK_STREAM ||
634 so->so_proto->pr_protocol != IPPROTO_TCP) {
635 fdrop(fp, curthread);
640 if (ic->ic_socket != NULL) {
642 fdrop(fp, curthread);
645 ic->ic_disconnecting = false;
647 fp->f_ops = &badfileops;
649 fdrop(fp, curthread);
652 /* Find the adapter offloading this socket. */
655 t4_iterate(find_offload_adapter, &fa);
659 ci = icc->sc->iscsi_ulp_softc;
664 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))
668 * socket could not have been "unoffloaded" if here.
670 MPASS(tp->t_flags & TF_TOE);
671 MPASS(tp->tod != NULL);
672 MPASS(tp->t_toe != NULL);
674 MPASS(toep->vi->pi->adapter == icc->sc);
676 icc->cwt = cxgbei_select_worker_thread(icc);
679 * We maintain the _send_ DSL in this field just to have a
680 * convenient way to assert that the kernel never sends
681 * oversized PDUs. This field is otherwise unused in the driver
684 ic->ic_max_data_segment_length = ci->max_tx_pdu_len -
687 icc->ulp_submode = 0;
688 if (ic->ic_header_crc32c) {
689 icc->ulp_submode |= ULP_CRC_HEADER;
690 ic->ic_max_data_segment_length -=
691 ISCSI_HEADER_DIGEST_SIZE;
693 if (ic->ic_data_crc32c) {
694 icc->ulp_submode |= ULP_CRC_DATA;
695 ic->ic_max_data_segment_length -=
696 ISCSI_DATA_DIGEST_SIZE;
698 so->so_options |= SO_NO_DDP;
699 toep->ulp_mode = ULP_MODE_ISCSI;
702 send_iscsi_flowc_wr(icc->sc, toep, ci->max_tx_pdu_len);
703 set_ulp_mode_iscsi(icc->sc, toep, ic->ic_header_crc32c,
710 error = icl_cxgbei_setsockopt(ic, so, ci->max_tx_pdu_len,
718 icl_cxgbei_conn_close(struct icl_conn *ic)
720 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
725 struct toepcb *toep = icc->toep;
727 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
728 ICL_CONN_LOCK_ASSERT_NOT(ic);
732 if (ic->ic_disconnecting || so == NULL) {
733 CTR4(KTR_CXGBE, "%s: icc %p (disconnecting = %d), so %p",
734 __func__, icc, ic->ic_disconnecting, so);
738 ic->ic_disconnecting = true;
740 /* These are unused in this driver right now. */
741 MPASS(STAILQ_EMPTY(&ic->ic_to_send));
742 MPASS(ic->ic_receive_pdu == NULL);
745 KASSERT(ic->ic_outstanding_pdus == 0,
746 ("destroying session with %d outstanding PDUs",
747 ic->ic_outstanding_pdus));
751 CTR3(KTR_CXGBE, "%s: tid %d, icc %p", __func__, toep ? toep->tid : -1,
756 if (toep != NULL) { /* NULL if connection was never offloaded. */
758 mbufq_drain(&toep->ulp_pduq);
760 if (icc->rx_flags & RXF_ACTIVE) {
761 volatile u_int *p = &icc->rx_flags;
766 while (*p & RXF_ACTIVE)
773 while (!STAILQ_EMPTY(&icc->rcvd_pdus)) {
774 ip = STAILQ_FIRST(&icc->rcvd_pdus);
775 STAILQ_REMOVE_HEAD(&icc->rcvd_pdus, ip_next);
776 icl_cxgbei_conn_pdu_free(ic, ip);
783 ic->ic_socket = NULL;
787 * XXXNP: we should send RST instead of FIN when PDUs held in various
788 * queues were purged instead of delivered reliably but soabort isn't
789 * really general purpose and wouldn't do the right thing here.
795 icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
796 struct ccb_scsiio *csio, uint32_t *ittp, void **arg)
798 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
799 struct toepcb *toep = icc->toep;
800 struct adapter *sc = icc->sc;
801 struct cxgbei_data *ci = sc->iscsi_ulp_softc;
802 struct ppod_region *pr = &ci->pr;
803 struct ppod_reservation *prsv;
807 /* This is for the offload driver's state. Must not be set already. */
811 if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN ||
812 csio->dxfer_len < ci->ddp_threshold) {
815 * No DDP for this I/O. Allocate an ITT (based on the one
816 * passed in) that cannot be a valid hardware DDP tag in the
819 itt = *ittp & M_PPOD_TAG;
820 itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit;
821 *ittp = htobe32(itt);
822 MPASS(*arg == NULL); /* State is maintained for DDP only. */
824 counter_u64_add(ci->ddp_setup_error, 1);
829 * Reserve resources for DDP, update the itt that should be used in the
830 * PDU, and save DDP specific state for this I/O in *arg.
833 prsv = uma_zalloc(prsv_zone, M_NOWAIT);
839 /* XXX add support for all CAM_DATA_ types */
840 MPASS((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR);
841 rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr,
842 csio->dxfer_len, prsv);
844 uma_zfree(prsv_zone, prsv);
848 rc = t4_write_page_pods_for_buf(sc, toep->ofld_txq, toep->tid, prsv,
849 (vm_offset_t)csio->data_ptr, csio->dxfer_len);
851 t4_free_page_pods(prsv);
852 uma_zfree(prsv_zone, prsv);
856 *ittp = htobe32(prsv->prsv_tag);
858 counter_u64_add(ci->ddp_setup_ok, 1);
863 icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg)
867 struct ppod_reservation *prsv = arg;
869 t4_free_page_pods(prsv);
870 uma_zfree(prsv_zone, prsv);
874 /* XXXNP: PDU should be passed in as parameter, like on the initiator. */
875 #define io_to_request_pdu(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr)
876 #define io_to_ppod_reservation(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr)
879 icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
880 uint32_t *tttp, void **arg)
882 struct icl_cxgbei_conn *icc = ic_to_icc(ic);
883 struct toepcb *toep = icc->toep;
884 struct ctl_scsiio *ctsio = &io->scsiio;
885 struct adapter *sc = icc->sc;
886 struct cxgbei_data *ci = sc->iscsi_ulp_softc;
887 struct ppod_region *pr = &ci->pr;
888 struct ppod_reservation *prsv;
890 int xferlen, rc = 0, alias;
892 /* This is for the offload driver's state. Must not be set already. */
896 if (ctsio->ext_data_filled == 0) {
898 struct icl_pdu *ip = io_to_request_pdu(io);
901 struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
903 MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
904 MPASS(ic == ip->ip_conn);
905 MPASS(ip->ip_bhs_mbuf != NULL);
907 first_burst = icl_pdu_data_segment_length(ip);
910 * Note that ICL calls conn_transfer_setup even if the first
911 * burst had everything and there's nothing left to transfer.
913 MPASS(ctsio->kern_data_len >= first_burst);
914 xferlen = ctsio->kern_data_len;
915 if (xferlen - first_burst < ci->ddp_threshold) {
918 * No DDP for this transfer. Allocate a TTT (based on
919 * the one passed in) that cannot be a valid hardware
920 * DDP tag in the iSCSI region.
922 ttt = *tttp & M_PPOD_TAG;
923 ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit;
924 *tttp = htobe32(ttt);
925 MPASS(io_to_ppod_reservation(io) == NULL);
927 counter_u64_add(ci->ddp_setup_error, 1);
931 if (ctsio->kern_sg_entries == 0)
932 buf = (vm_offset_t)ctsio->kern_data_ptr;
933 else if (ctsio->kern_sg_entries == 1) {
934 struct ctl_sg_entry *sgl = (void *)ctsio->kern_data_ptr;
936 MPASS(sgl->len == xferlen);
937 buf = (vm_offset_t)sgl->addr;
939 rc = EAGAIN; /* XXX implement */
945 * Reserve resources for DDP, update the ttt that should be used
946 * in the PDU, and save DDP specific state for this I/O.
949 MPASS(io_to_ppod_reservation(io) == NULL);
950 prsv = uma_zalloc(prsv_zone, M_NOWAIT);
956 rc = t4_alloc_page_pods_for_buf(pr, buf, xferlen, prsv);
958 uma_zfree(prsv_zone, prsv);
962 rc = t4_write_page_pods_for_buf(sc, toep->ofld_txq, toep->tid,
965 t4_free_page_pods(prsv);
966 uma_zfree(prsv_zone, prsv);
970 *tttp = htobe32(prsv->prsv_tag);
971 io_to_ppod_reservation(io) = prsv;
973 counter_u64_add(ci->ddp_setup_ok, 1);
978 * In the middle of an I/O. A non-NULL page pod reservation indicates
979 * that a DDP buffer is being used for the I/O.
982 prsv = io_to_ppod_reservation(ctsio);
986 alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift;
988 prsv->prsv_tag &= ~pr->pr_alias_mask;
989 prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask;
991 *tttp = htobe32(prsv->prsv_tag);
998 icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg)
1000 struct ctl_scsiio *ctsio = arg;
1002 if (ctsio != NULL && ctsio->kern_data_len == ctsio->ext_data_filled) {
1003 struct ppod_reservation *prsv;
1005 prsv = io_to_ppod_reservation(ctsio);
1006 MPASS(prsv != NULL);
1008 t4_free_page_pods(prsv);
1009 uma_zfree(prsv_zone, prsv);
1014 cxgbei_limits(struct adapter *sc, void *arg)
1016 struct icl_drv_limits *idl = arg;
1017 struct cxgbei_data *ci;
1020 if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4lims") != 0)
1023 if (uld_active(sc, ULD_ISCSI)) {
1024 ci = sc->iscsi_ulp_softc;
1028 * AHS is not supported by the kernel so we'll not account for
1029 * it either in our PDU len -> data segment len conversions.
1032 max_dsl = ci->max_rx_pdu_len - ISCSI_BHS_SIZE -
1033 ISCSI_HEADER_DIGEST_SIZE - ISCSI_DATA_DIGEST_SIZE;
1034 if (idl->idl_max_recv_data_segment_length > max_dsl)
1035 idl->idl_max_recv_data_segment_length = max_dsl;
1037 max_dsl = ci->max_tx_pdu_len - ISCSI_BHS_SIZE -
1038 ISCSI_HEADER_DIGEST_SIZE - ISCSI_DATA_DIGEST_SIZE;
1039 if (idl->idl_max_send_data_segment_length > max_dsl)
1040 idl->idl_max_send_data_segment_length = max_dsl;
1043 end_synchronized_op(sc, LOCK_HELD);
1047 icl_cxgbei_limits(struct icl_drv_limits *idl)
1050 /* Maximum allowed by the RFC. cxgbei_limits will clip them. */
1051 idl->idl_max_recv_data_segment_length = (1 << 24) - 1;
1052 idl->idl_max_send_data_segment_length = (1 << 24) - 1;
1054 /* These are somewhat arbitrary. */
1055 idl->idl_max_burst_length = 2 * 1024 * 1024;
1056 idl->idl_first_burst_length = 8192;
1058 t4_iterate(cxgbei_limits, idl);
1064 icl_cxgbei_mod_load(void)
1069 * Space to track pagepod reservations.
1071 prsv_zone = uma_zcreate("Pagepod reservations",
1072 sizeof(struct ppod_reservation), NULL, NULL, NULL, NULL,
1073 UMA_ALIGN_CACHE, 0);
1075 refcount_init(&icl_cxgbei_ncons, 0);
1077 rc = icl_register("cxgbei", false, -100, icl_cxgbei_limits,
1078 icl_cxgbei_new_conn);
1084 icl_cxgbei_mod_unload(void)
1087 if (icl_cxgbei_ncons != 0)
1090 icl_unregister("cxgbei", false);
1092 uma_zdestroy(prsv_zone);