2 * Copyright (c) 2012 Chelsio Communications, Inc.
5 * Chelsio T5xx iSCSI driver
7 * Written by: Sreenivasa Honnur <shonnur@chelsio.com>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
35 #include "opt_inet6.h"
37 #include <sys/types.h>
38 #include <sys/param.h>
39 #include <sys/kernel.h>
40 #include <sys/module.h>
41 #include <sys/systm.h>
44 #include <sys/errno.h>
45 #include <sys/kthread.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
51 #include <sys/mutex.h>
52 #include <sys/condvar.h>
54 #include <netinet/in.h>
55 #include <netinet/in_pcb.h>
56 #include <netinet/toecore.h>
57 #include <netinet/tcp_var.h>
58 #include <netinet/tcp_fsm.h>
60 #include <cam/scsi/scsi_all.h>
61 #include <cam/scsi/scsi_da.h>
62 #include <cam/ctl/ctl_io.h>
63 #include <cam/ctl/ctl.h>
64 #include <cam/ctl/ctl_backend.h>
65 #include <cam/ctl/ctl_error.h>
66 #include <cam/ctl/ctl_frontend.h>
67 #include <cam/ctl/ctl_debug.h>
68 #include <cam/ctl/ctl_ha.h>
69 #include <cam/ctl/ctl_ioctl.h>
71 #include <dev/iscsi/icl.h>
72 #include <dev/iscsi/iscsi_proto.h>
73 #include <dev/iscsi/iscsi_ioctl.h>
74 #include <dev/iscsi/iscsi.h>
75 #include <cam/ctl/ctl_frontend_iscsi.h>
78 #include <cam/cam_ccb.h>
79 #include <cam/cam_xpt.h>
80 #include <cam/cam_debug.h>
81 #include <cam/cam_sim.h>
82 #include <cam/cam_xpt_sim.h>
83 #include <cam/cam_xpt_periph.h>
84 #include <cam/cam_periph.h>
85 #include <cam/cam_compat.h>
86 #include <cam/scsi/scsi_message.h>
88 #include "common/common.h"
89 #include "common/t4_msg.h"
90 #include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */
91 #include "tom/t4_tom.h"
94 static int worker_thread_count;
95 static struct cxgbei_worker_thread_softc *cwt_softc;
96 static struct proc *cxgbei_proc;
98 /* XXXNP some header instead. */
99 struct icl_pdu *icl_cxgbei_new_pdu(int);
100 void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *);
101 void icl_cxgbei_conn_pdu_free(struct icl_conn *, struct icl_pdu *);
104 free_ci_counters(struct cxgbei_data *ci)
107 #define FREE_CI_COUNTER(x) do { \
108 if (ci->x != NULL) { \
109 counter_u64_free(ci->x); \
114 FREE_CI_COUNTER(ddp_setup_ok);
115 FREE_CI_COUNTER(ddp_setup_error);
116 FREE_CI_COUNTER(ddp_bytes);
117 FREE_CI_COUNTER(ddp_pdus);
118 FREE_CI_COUNTER(fl_bytes);
119 FREE_CI_COUNTER(fl_pdus);
120 #undef FREE_CI_COUNTER
124 alloc_ci_counters(struct cxgbei_data *ci)
127 #define ALLOC_CI_COUNTER(x) do { \
128 ci->x = counter_u64_alloc(M_WAITOK); \
133 ALLOC_CI_COUNTER(ddp_setup_ok);
134 ALLOC_CI_COUNTER(ddp_setup_error);
135 ALLOC_CI_COUNTER(ddp_bytes);
136 ALLOC_CI_COUNTER(ddp_pdus);
137 ALLOC_CI_COUNTER(fl_bytes);
138 ALLOC_CI_COUNTER(fl_pdus);
139 #undef ALLOC_CI_COUNTER
143 free_ci_counters(ci);
148 read_pdu_limits(struct adapter *sc, uint32_t *max_tx_pdu_len,
149 uint32_t *max_rx_pdu_len)
151 uint32_t tx_len, rx_len, r, v;
153 rx_len = t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE);
154 tx_len = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
156 r = t4_read_reg(sc, A_TP_PARA_REG2);
157 rx_len = min(rx_len, G_MAXRXDATA(r));
158 tx_len = min(tx_len, G_MAXRXDATA(r));
160 r = t4_read_reg(sc, A_TP_PARA_REG7);
161 v = min(G_PMMAXXFERLEN0(r), G_PMMAXXFERLEN1(r));
162 rx_len = min(rx_len, v);
163 tx_len = min(tx_len, v);
165 /* Remove after FW_FLOWC_MNEM_TXDATAPLEN_MAX fix in firmware. */
166 tx_len = min(tx_len, 3 * 4096);
168 *max_tx_pdu_len = rounddown2(tx_len, 512);
169 *max_rx_pdu_len = rounddown2(rx_len, 512);
173 * Initialize the software state of the iSCSI ULP driver.
175 * ENXIO means firmware didn't set up something that it was supposed to.
178 cxgbei_init(struct adapter *sc, struct cxgbei_data *ci)
180 struct sysctl_oid *oid;
181 struct sysctl_oid_list *children;
182 struct ppod_region *pr;
186 MPASS(sc->vres.iscsi.size > 0);
189 rc = alloc_ci_counters(ci);
193 read_pdu_limits(sc, &ci->max_tx_pdu_len, &ci->max_rx_pdu_len);
196 r = t4_read_reg(sc, A_ULP_RX_ISCSI_PSZ);
197 rc = t4_init_ppod_region(pr, &sc->vres.iscsi, r, "iSCSI page pods");
199 device_printf(sc->dev,
200 "%s: failed to initialize the iSCSI page pod region: %u.\n",
202 free_ci_counters(ci);
206 r = t4_read_reg(sc, A_ULP_RX_ISCSI_TAGMASK);
207 r &= V_ISCSITAGMASK(M_ISCSITAGMASK);
208 if (r != pr->pr_tag_mask) {
210 * Recent firmwares are supposed to set up the iSCSI tagmask
211 * but we'll do it ourselves it the computed value doesn't match
212 * what's in the register.
214 device_printf(sc->dev,
215 "tagmask 0x%08x does not match computed mask 0x%08x.\n", r,
217 t4_set_reg_field(sc, A_ULP_RX_ISCSI_TAGMASK,
218 V_ISCSITAGMASK(M_ISCSITAGMASK), pr->pr_tag_mask);
221 sysctl_ctx_init(&ci->ctx);
222 oid = device_get_sysctl_tree(sc->dev); /* dev.t5nex.X */
223 children = SYSCTL_CHILDREN(oid);
225 oid = SYSCTL_ADD_NODE(&ci->ctx, children, OID_AUTO, "iscsi", CTLFLAG_RD,
226 NULL, "iSCSI ULP statistics");
227 children = SYSCTL_CHILDREN(oid);
229 SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_setup_ok",
230 CTLFLAG_RD, &ci->ddp_setup_ok,
231 "# of times DDP buffer was setup successfully.");
233 SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_setup_error",
234 CTLFLAG_RD, &ci->ddp_setup_error,
235 "# of times DDP buffer setup failed.");
237 SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_bytes",
238 CTLFLAG_RD, &ci->ddp_bytes, "# of bytes placed directly");
240 SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_pdus",
241 CTLFLAG_RD, &ci->ddp_pdus, "# of PDUs with data placed directly.");
243 SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "fl_bytes",
244 CTLFLAG_RD, &ci->fl_bytes, "# of data bytes delivered in freelist");
246 SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "fl_pdus",
247 CTLFLAG_RD, &ci->fl_pdus,
248 "# of PDUs with data delivered in freelist");
250 ci->ddp_threshold = 2048;
251 SYSCTL_ADD_UINT(&ci->ctx, children, OID_AUTO, "ddp_threshold",
252 CTLFLAG_RW, &ci->ddp_threshold, 0, "Rx zero copy threshold");
258 do_rx_iscsi_hdr(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
260 struct adapter *sc = iq->adapter;
261 struct cpl_iscsi_hdr *cpl = mtod(m, struct cpl_iscsi_hdr *);
262 u_int tid = GET_TID(cpl);
263 struct toepcb *toep = lookup_tid(sc, tid);
265 struct icl_cxgbei_pdu *icp;
266 uint16_t len_ddp = be16toh(cpl->pdu_len_ddp);
267 uint16_t len = be16toh(cpl->len);
270 MPASS(m->m_pkthdr.len == len + sizeof(*cpl));
272 ip = icl_cxgbei_new_pdu(M_NOWAIT);
274 CXGBE_UNIMPLEMENTED("PDU allocation failure");
275 m_copydata(m, sizeof(*cpl), ISCSI_BHS_SIZE, (caddr_t)ip->ip_bhs);
276 ip->ip_data_len = G_ISCSI_PDU_LEN(len_ddp) - len;
278 icp->icp_seq = ntohl(cpl->seq);
279 icp->icp_flags = ICPF_RX_HDR;
281 /* This is the start of a new PDU. There should be no old state. */
282 MPASS(toep->ulpcb2 == NULL);
286 CTR5(KTR_CXGBE, "%s: tid %u, cpl->len %u, pdu_len_ddp 0x%04x, icp %p",
287 __func__, tid, len, len_ddp, icp);
295 do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
297 struct adapter *sc = iq->adapter;
298 struct cxgbei_data *ci = sc->iscsi_ulp_softc;
299 struct cpl_iscsi_data *cpl = mtod(m, struct cpl_iscsi_data *);
300 u_int tid = GET_TID(cpl);
301 struct toepcb *toep = lookup_tid(sc, tid);
302 struct icl_cxgbei_pdu *icp = toep->ulpcb2;
305 MPASS(m->m_pkthdr.len == be16toh(cpl->len) + sizeof(*cpl));
307 /* Must already have received the header (but not the data). */
309 MPASS(icp->icp_flags == ICPF_RX_HDR);
310 MPASS(icp->ip.ip_data_mbuf == NULL);
313 m_adj(m, sizeof(*cpl));
314 MPASS(icp->ip.ip_data_len == m->m_pkthdr.len);
316 icp->icp_flags |= ICPF_RX_FLBUF;
317 icp->ip.ip_data_mbuf = m;
318 counter_u64_add(ci->fl_pdus, 1);
319 counter_u64_add(ci->fl_bytes, m->m_pkthdr.len);
322 CTR3(KTR_CXGBE, "%s: tid %u, cpl->len %u", __func__, tid,
330 do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
332 struct adapter *sc = iq->adapter;
333 struct cxgbei_data *ci = sc->iscsi_ulp_softc;
334 const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
335 u_int tid = GET_TID(cpl);
336 struct toepcb *toep = lookup_tid(sc, tid);
337 struct inpcb *inp = toep->inp;
341 struct icl_cxgbei_conn *icc;
343 struct icl_cxgbei_pdu *icp = toep->ulpcb2;
349 /* Must already be assembling a PDU. */
351 MPASS(icp->icp_flags & ICPF_RX_HDR); /* Data is optional. */
352 MPASS((icp->icp_flags & ICPF_RX_STATUS) == 0);
354 pdu_len = be16toh(cpl->len); /* includes everything. */
355 val = be32toh(cpl->ddpvld);
359 "%s: tid %u, cpl->len %u, ddpvld 0x%08x, icp_flags 0x%08x",
360 __func__, tid, pdu_len, val, icp->icp_flags);
363 icp->icp_flags |= ICPF_RX_STATUS;
365 if (val & F_DDP_PADDING_ERR)
366 icp->icp_flags |= ICPF_PAD_ERR;
367 if (val & F_DDP_HDRCRC_ERR)
368 icp->icp_flags |= ICPF_HCRC_ERR;
369 if (val & F_DDP_DATACRC_ERR)
370 icp->icp_flags |= ICPF_DCRC_ERR;
371 if (val & F_DDP_PDU && ip->ip_data_mbuf == NULL) {
372 MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
373 MPASS(ip->ip_data_len > 0);
374 icp->icp_flags |= ICPF_RX_DDP;
375 counter_u64_add(ci->ddp_pdus, 1);
376 counter_u64_add(ci->ddp_bytes, ip->ip_data_len);
380 if (__predict_false(inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT))) {
381 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
382 __func__, tid, pdu_len, inp->inp_flags);
384 icl_cxgbei_conn_pdu_free(NULL, ip);
392 MPASS(icp->icp_seq == tp->rcv_nxt);
393 MPASS(tp->rcv_wnd >= pdu_len);
394 tp->rcv_nxt += pdu_len;
395 tp->rcv_wnd -= pdu_len;
396 tp->t_rcvtime = ticks;
398 /* update rx credits */
399 toep->rx_credits += pdu_len;
400 t4_rcvd(&toep->td->tod, tp); /* XXX: sc->tom_softc.tod */
402 so = inp->inp_socket;
407 if (__predict_false(icc == NULL || sb->sb_state & SBS_CANTRCVMORE)) {
409 "%s: tid %u, excess rx (%d bytes), icc %p, sb_state 0x%x",
410 __func__, tid, pdu_len, icc, sb->sb_state);
414 INP_INFO_RLOCK(&V_tcbinfo);
416 tp = tcp_drop(tp, ECONNRESET);
419 INP_INFO_RUNLOCK(&V_tcbinfo);
421 icl_cxgbei_conn_pdu_free(NULL, ip);
427 MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
429 icl_cxgbei_new_pdu_set_conn(ip, ic);
431 MPASS(m == NULL); /* was unused, we'll use it now. */
432 m = sbcut_locked(sb, sbused(sb)); /* XXXNP: toep->sb_cc accounting? */
433 if (__predict_false(m != NULL)) {
434 int len = m_length(m, NULL);
437 * PDUs were received before the tid transitioned to ULP mode.
438 * Convert them to icl_cxgbei_pdus and send them to ICL before
441 CTR3(KTR_CXGBE, "%s: tid %u, %u bytes in so_rcv", __func__, tid,
444 /* XXXNP: needs to be rewritten. */
445 if (len == sizeof(struct iscsi_bhs) || len == 4 + sizeof(struct
447 struct icl_cxgbei_pdu *icp0;
450 ip0 = icl_cxgbei_new_pdu(M_NOWAIT);
451 icl_cxgbei_new_pdu_set_conn(ip0, ic);
453 CXGBE_UNIMPLEMENTED("PDU allocation failure");
454 icp0 = ip_to_icp(ip0);
455 icp0->icp_seq = 0; /* XXX */
456 icp0->icp_flags = ICPF_RX_HDR | ICPF_RX_STATUS;
457 m_copydata(m, 0, sizeof(struct iscsi_bhs), (void *)ip0->ip_bhs);
458 STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip0, ip_next);
463 STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
464 if ((icc->rx_flags & RXF_ACTIVE) == 0) {
465 struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt];
467 mtx_lock(&cwt->cwt_lock);
468 icc->rx_flags |= RXF_ACTIVE;
469 TAILQ_INSERT_TAIL(&cwt->rx_head, icc, rx_link);
470 if (cwt->cwt_state == CWT_SLEEPING) {
471 cwt->cwt_state = CWT_RUNNING;
472 cv_signal(&cwt->cwt_cv);
474 mtx_unlock(&cwt->cwt_lock);
487 cxgbei_activate(struct adapter *sc)
489 struct cxgbei_data *ci;
492 ASSERT_SYNCHRONIZED_OP(sc);
494 if (uld_active(sc, ULD_ISCSI)) {
495 KASSERT(0, ("%s: iSCSI offload already enabled on adapter %p",
500 if (sc->iscsicaps == 0 || sc->vres.iscsi.size == 0) {
501 device_printf(sc->dev,
502 "not iSCSI offload capable, or capability disabled.\n");
506 /* per-adapter softc for iSCSI */
507 ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_WAITOK);
511 rc = cxgbei_init(sc, ci);
517 sc->iscsi_ulp_softc = ci;
523 cxgbei_deactivate(struct adapter *sc)
525 struct cxgbei_data *ci = sc->iscsi_ulp_softc;
527 ASSERT_SYNCHRONIZED_OP(sc);
530 sysctl_ctx_free(&ci->ctx);
531 t4_free_ppod_region(&ci->pr);
532 free_ci_counters(ci);
534 sc->iscsi_ulp_softc = NULL;
541 cxgbei_activate_all(struct adapter *sc, void *arg __unused)
544 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isact") != 0)
547 /* Activate iSCSI if any port on this adapter has IFCAP_TOE enabled. */
548 if (sc->offload_map && !uld_active(sc, ULD_ISCSI))
549 (void) t4_activate_uld(sc, ULD_ISCSI);
551 end_synchronized_op(sc, 0);
555 cxgbei_deactivate_all(struct adapter *sc, void *arg __unused)
558 if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4isdea") != 0)
561 if (uld_active(sc, ULD_ISCSI))
562 (void) t4_deactivate_uld(sc, ULD_ISCSI);
564 end_synchronized_op(sc, 0);
567 static struct uld_info cxgbei_uld_info = {
569 .activate = cxgbei_activate,
570 .deactivate = cxgbei_deactivate,
576 struct cxgbei_worker_thread_softc *cwt = arg;
577 struct icl_cxgbei_conn *icc = NULL;
581 STAILQ_HEAD(, icl_pdu) rx_pdus = STAILQ_HEAD_INITIALIZER(rx_pdus);
585 mtx_lock(&cwt->cwt_lock);
586 MPASS(cwt->cwt_state == 0);
587 cwt->cwt_state = CWT_RUNNING;
588 cv_signal(&cwt->cwt_cv);
590 while (__predict_true(cwt->cwt_state != CWT_STOP)) {
591 cwt->cwt_state = CWT_RUNNING;
592 while ((icc = TAILQ_FIRST(&cwt->rx_head)) != NULL) {
593 TAILQ_REMOVE(&cwt->rx_head, icc, rx_link);
594 mtx_unlock(&cwt->cwt_lock);
597 sb = &ic->ic_socket->so_rcv;
600 MPASS(icc->rx_flags & RXF_ACTIVE);
601 if (__predict_true(!(sb->sb_state & SBS_CANTRCVMORE))) {
602 MPASS(STAILQ_EMPTY(&rx_pdus));
603 STAILQ_SWAP(&icc->rcvd_pdus, &rx_pdus, icl_pdu);
606 /* Hand over PDUs to ICL. */
607 while ((ip = STAILQ_FIRST(&rx_pdus)) != NULL) {
608 STAILQ_REMOVE_HEAD(&rx_pdus, ip_next);
613 MPASS(STAILQ_EMPTY(&rx_pdus));
615 MPASS(icc->rx_flags & RXF_ACTIVE);
616 if (STAILQ_EMPTY(&icc->rcvd_pdus) ||
617 __predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
618 icc->rx_flags &= ~RXF_ACTIVE;
621 * More PDUs were received while we were busy
622 * handing over the previous batch to ICL.
623 * Re-add this connection to the end of the
626 mtx_lock(&cwt->cwt_lock);
627 TAILQ_INSERT_TAIL(&cwt->rx_head, icc,
629 mtx_unlock(&cwt->cwt_lock);
633 mtx_lock(&cwt->cwt_lock);
636 /* Inner loop doesn't check for CWT_STOP, do that first. */
637 if (__predict_false(cwt->cwt_state == CWT_STOP))
639 cwt->cwt_state = CWT_SLEEPING;
640 cv_wait(&cwt->cwt_cv, &cwt->cwt_lock);
643 MPASS(TAILQ_FIRST(&cwt->rx_head) == NULL);
644 mtx_assert(&cwt->cwt_lock, MA_OWNED);
645 cwt->cwt_state = CWT_STOPPED;
646 cv_signal(&cwt->cwt_cv);
647 mtx_unlock(&cwt->cwt_lock);
652 start_worker_threads(void)
655 struct cxgbei_worker_thread_softc *cwt;
657 worker_thread_count = min(mp_ncpus, 32);
658 cwt_softc = malloc(worker_thread_count * sizeof(*cwt), M_CXGBE,
661 MPASS(cxgbei_proc == NULL);
662 for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) {
663 mtx_init(&cwt->cwt_lock, "cwt lock", NULL, MTX_DEF);
664 cv_init(&cwt->cwt_cv, "cwt cv");
665 TAILQ_INIT(&cwt->rx_head);
666 rc = kproc_kthread_add(cwt_main, cwt, &cxgbei_proc, NULL, 0, 0,
669 printf("cxgbei: failed to start thread #%d/%d (%d)\n",
670 i + 1, worker_thread_count, rc);
671 mtx_destroy(&cwt->cwt_lock);
672 cv_destroy(&cwt->cwt_cv);
673 bzero(&cwt, sizeof(*cwt));
675 free(cwt_softc, M_CXGBE);
676 worker_thread_count = 0;
681 /* Not fatal, carry on with fewer threads. */
682 worker_thread_count = i;
687 /* Wait for thread to start before moving on to the next one. */
688 mtx_lock(&cwt->cwt_lock);
689 while (cwt->cwt_state == 0)
690 cv_wait(&cwt->cwt_cv, &cwt->cwt_lock);
691 mtx_unlock(&cwt->cwt_lock);
694 MPASS(cwt_softc != NULL);
695 MPASS(worker_thread_count > 0);
700 stop_worker_threads(void)
703 struct cxgbei_worker_thread_softc *cwt = &cwt_softc[0];
705 MPASS(worker_thread_count >= 0);
707 for (i = 0, cwt = &cwt_softc[0]; i < worker_thread_count; i++, cwt++) {
708 mtx_lock(&cwt->cwt_lock);
709 MPASS(cwt->cwt_state == CWT_RUNNING ||
710 cwt->cwt_state == CWT_SLEEPING);
711 cwt->cwt_state = CWT_STOP;
712 cv_signal(&cwt->cwt_cv);
714 cv_wait(&cwt->cwt_cv, &cwt->cwt_lock);
715 } while (cwt->cwt_state != CWT_STOPPED);
716 mtx_unlock(&cwt->cwt_lock);
718 free(cwt_softc, M_CXGBE);
721 /* Select a worker thread for a connection. */
723 cxgbei_select_worker_thread(struct icl_cxgbei_conn *icc)
725 struct adapter *sc = icc->sc;
726 struct toepcb *toep = icc->toep;
729 n = worker_thread_count / sc->sge.nofldrxq;
731 i = toep->vi->pi->port_id * n + arc4random() % n;
733 i = arc4random() % worker_thread_count;
735 CTR3(KTR_CXGBE, "%s: tid %u, cwt %u", __func__, toep->tid, i);
741 cxgbei_mod_load(void)
745 t4_register_cpl_handler(CPL_ISCSI_HDR, do_rx_iscsi_hdr);
746 t4_register_cpl_handler(CPL_ISCSI_DATA, do_rx_iscsi_data);
747 t4_register_cpl_handler(CPL_RX_ISCSI_DDP, do_rx_iscsi_ddp);
749 rc = start_worker_threads();
753 rc = t4_register_uld(&cxgbei_uld_info);
755 stop_worker_threads();
759 t4_iterate(cxgbei_activate_all, NULL);
765 cxgbei_mod_unload(void)
768 t4_iterate(cxgbei_deactivate_all, NULL);
770 if (t4_unregister_uld(&cxgbei_uld_info) == EBUSY)
773 stop_worker_threads();
775 t4_register_cpl_handler(CPL_ISCSI_HDR, NULL);
776 t4_register_cpl_handler(CPL_ISCSI_DATA, NULL);
777 t4_register_cpl_handler(CPL_RX_ISCSI_DDP, NULL);
784 cxgbei_modevent(module_t mod, int cmd, void *arg)
791 rc = cxgbei_mod_load();
793 rc = icl_cxgbei_mod_load();
797 rc = icl_cxgbei_mod_unload();
799 rc = cxgbei_mod_unload();
806 printf("cxgbei: compiled without TCP_OFFLOAD support.\n");
813 static moduledata_t cxgbei_mod = {
819 MODULE_VERSION(cxgbei, 1);
820 DECLARE_MODULE(cxgbei, cxgbei_mod, SI_SUB_EXEC, SI_ORDER_ANY);
821 MODULE_DEPEND(cxgbei, t4_tom, 1, 1, 1);
822 MODULE_DEPEND(cxgbei, cxgbe, 1, 1, 1);
823 MODULE_DEPEND(cxgbei, icl, 1, 1, 1);